Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-code into master-nj
Co-authored-by: Cursor <cursoragent@cursor.com> # Conflicts: # frontend-web/src/components/workspace/workspace-nav-chat-list.tsx # frontend-web/src/pages/WorkspaceRoutes.tsx # offline-backend-20260512/backend/app/gateway/app.py # offline-backend-20260512/backend/app/gateway/deps.py # offline-backend-20260512/backend/pyproject.toml
This commit is contained in:
commit
cf9fe58905
109 files changed
+18863
-982
No files matched your search
@@ -32,6 +32,7 @@ from app.gateway.routers import (
|
||||
agents,
|
||||
ai_writing,
|
||||
artifacts,
|
||||
assistant_knowledge,
|
||||
assistants_compat,
|
||||
auth,
|
||||
browser_context,
|
||||
@@ -88,6 +89,7 @@ from app.gateway.routers import (
|
||||
sensitive_words,
|
||||
sentiment_agent,
|
||||
skill_addresses,
|
||||
skill_knowledge,
|
||||
skills,
|
||||
suggestions,
|
||||
system_settings,
|
||||
@@ -1581,6 +1583,13 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
# Skills API is mounted at /api/skills
|
||||
app.include_router(skills.router)
|
||||
# Admin skill distillation matrix + global assistant knowledge base.
|
||||
app.include_router(skill_knowledge.router)
|
||||
app.include_router(assistant_knowledge.router)
|
||||
app.include_router(assistant_knowledge.export_router)
|
||||
from app.gateway.routers import wiki_packages
|
||||
|
||||
app.include_router(wiki_packages.router)
|
||||
|
||||
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
|
||||
app.include_router(artifacts.router)
|
||||
|
||||
@@ -354,6 +354,33 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.llmwiki_store = make_llmwiki_store(sf)
|
||||
register_llmwiki_store(app.state.llmwiki_store)
|
||||
|
||||
# Skill distillation and the single global assistant knowledge base.
|
||||
# Both repositories live in ``deerflow.*``; only this composition layer
|
||||
# wires them to Gateway executors and remote adapters.
|
||||
from deerflow.persistence.assistant_knowledge import make_assistant_knowledge_store
|
||||
from deerflow.persistence.skill_knowledge import make_skill_knowledge_store
|
||||
|
||||
app.state.assistant_knowledge_store = make_assistant_knowledge_store(sf)
|
||||
app.state.skill_knowledge_store = make_skill_knowledge_store(sf)
|
||||
if app.state.skill_knowledge_store is not None and app.state.assistant_knowledge_store is not None:
|
||||
from app.gateway.skill_knowledge_job_dispatcher import SkillKnowledgeJobDispatcher
|
||||
from app.gateway.skill_knowledge_job_executor import SkillKnowledgeJobExecutor
|
||||
|
||||
app.state.skill_knowledge_executor = SkillKnowledgeJobExecutor(app)
|
||||
app.state.skill_knowledge_dispatcher = SkillKnowledgeJobDispatcher(
|
||||
app.state.skill_knowledge_store,
|
||||
app.state.skill_knowledge_executor,
|
||||
)
|
||||
app.state.skill_knowledge_dispatcher.start()
|
||||
from app.gateway.skill_knowledge_auto_sync import SkillKnowledgeAutoSyncService
|
||||
|
||||
app.state.skill_knowledge_auto_sync = SkillKnowledgeAutoSyncService(app)
|
||||
app.state.skill_knowledge_auto_sync.start()
|
||||
else:
|
||||
app.state.skill_knowledge_executor = None
|
||||
app.state.skill_knowledge_dispatcher = None
|
||||
app.state.skill_knowledge_auto_sync = None
|
||||
|
||||
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
|
||||
# migrations and status inspection work while feature flags are off;
|
||||
# network/search services only exist when explicitly enabled.
|
||||
@@ -374,13 +401,32 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
|
||||
weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
|
||||
if local_index_config.enabled:
|
||||
embedding_config = local_index_config.embedding
|
||||
# The administrator may select or upload the offline encoder from the
|
||||
# UI. This runtime-owned value deliberately overrides config.yaml and
|
||||
# survives application restarts without modifying release assets.
|
||||
if embedding_config.provider == "local":
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
|
||||
runtime_model_path = load_system_settings().local_wiki_encoder.model_path.strip()
|
||||
if runtime_model_path:
|
||||
embedding_config = embedding_config.model_copy(
|
||||
update={"local_model_path": runtime_model_path}
|
||||
)
|
||||
embedding_configured = bool(
|
||||
embedding_config.enabled
|
||||
and (embedding_config.base_url or embedding_config.provider == "local")
|
||||
and embedding_config.model.strip()
|
||||
)
|
||||
if embedding_configured:
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
|
||||
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(embedding_config)
|
||||
if local_index_config.enabled and app.state.llmwiki_embedding is not None:
|
||||
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
|
||||
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
|
||||
app.state.llmwiki_vector_cache = WikiVectorCache(
|
||||
app.state.llmwiki_index_store,
|
||||
max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
|
||||
@@ -399,6 +445,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.llmwiki_vector_cache,
|
||||
weknora_client,
|
||||
)
|
||||
from app.gateway.knowledge_transfer import sync_to_global
|
||||
|
||||
async def on_wiki_vectorized(mapping):
|
||||
return await sync_to_global(app, mapping)
|
||||
|
||||
app.state.llmwiki_sync_service.on_completed = on_wiki_vectorized
|
||||
if weknora_client is not None or app.state.llmwiki_vector_search is not None:
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
|
||||
|
||||
@@ -676,6 +728,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
from deerflow.workflows.runtime.sql_runner import dispose_engines
|
||||
|
||||
await dispose_engines()
|
||||
skill_knowledge_auto_sync = getattr(app.state, "skill_knowledge_auto_sync", None)
|
||||
if skill_knowledge_auto_sync is not None:
|
||||
await skill_knowledge_auto_sync.close()
|
||||
skill_knowledge_dispatcher = getattr(app.state, "skill_knowledge_dispatcher", None)
|
||||
if skill_knowledge_dispatcher is not None:
|
||||
await skill_knowledge_dispatcher.close()
|
||||
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
|
||||
if queue is not None:
|
||||
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Wiki package flow orchestration; all remote writes use WeKnora APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from starlette.requests import Request
|
||||
|
||||
|
||||
async def sync_to_global(app, mapping):
|
||||
from app.gateway.routers.assistant_knowledge import _export_weknora_then_import_for_job
|
||||
|
||||
if mapping.get("is_conversation_deposit") or mapping.get("owner_user_id") == "system":
|
||||
return {"status": "excluded"}
|
||||
store = app.state.assistant_knowledge_store
|
||||
revision, fingerprint, _ = await app.state.llmwiki_index_store.get_index_revision(str(mapping["id"]))
|
||||
base = await store.initialize(created_by=None)
|
||||
for source in await store.list_sources(base_id=base["id"]):
|
||||
meta = source.get("metadata") or {}
|
||||
if source.get("source_key") == mapping["id"] and meta.get("index_revision") == revision:
|
||||
job = await store.get_job(source.get("last_import_job_id")) if source.get("last_import_job_id") else None
|
||||
if job and job.get("status") == "completed":
|
||||
return {"status": "unchanged", "job_id": job["id"], "base_id": base["id"]}
|
||||
source = {"provider": "weknora", "type": "weknora", "knowledge_base_id": mapping["id"], "knowledge_base_name": mapping["name"], "index_revision": revision, "embedding_fingerprint": fingerprint}
|
||||
job = await store.create_queued_import_job(source_type="weknora", source_key=mapping["id"], source_name=mapping["name"], trigger="vectorization_completed", created_by=None, metadata=source, mapping_id=mapping["id"], base_id=base["id"])
|
||||
await _export_weknora_then_import_for_job(request=Request({"type": "http", "app": app}), job_id=job["id"], mapping=mapping, remote_id=mapping["weknora_id"], source=source, actor=None, index_fresh=True)
|
||||
result = await store.get_job(job["id"])
|
||||
if not result or result.get("status") != "completed":
|
||||
raise RuntimeError(f"向量化已完成,但知识梳理同步失败:{(result or {}).get('last_error_message') or job['id']}")
|
||||
return {"status": "completed", "job_id": job["id"], "base_id": base["id"], "counts": result.get("counts")}
|
||||
@@ -49,9 +49,14 @@ async def _sync_once(app: FastAPI) -> None:
|
||||
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
|
||||
interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
|
||||
logger.info(
|
||||
"Local Wiki auto-sync scheduler started; first scan delayed by %s seconds",
|
||||
"Local Wiki auto-sync scheduler started; first scan is immediate, interval=%s seconds",
|
||||
interval,
|
||||
)
|
||||
while True:
|
||||
try:
|
||||
await _sync_once(app)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Local Wiki auto-sync scheduler tick failed (non-fatal)")
|
||||
await asyncio.sleep(interval)
|
||||
await _sync_once(app)
|
||||
@@ -40,6 +40,7 @@ _MAX_SOURCE_CHARS = 1400
|
||||
@dataclass(frozen=True)
|
||||
class LlmWikiRagContext:
|
||||
knowledge_base_ids: list[str]
|
||||
assistant_knowledge_base_ids: list[str]
|
||||
source_count: int
|
||||
prompt: str
|
||||
sources: list[dict[str, Any]]
|
||||
@@ -50,6 +51,10 @@ def _store(request: Request) -> LlmWikiStore | None:
|
||||
return getattr(request.app.state, "llmwiki_store", None)
|
||||
|
||||
|
||||
def _assistant_store(request: Request) -> Any | None:
|
||||
return getattr(request.app.state, "assistant_knowledge_store", None)
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
@@ -76,6 +81,13 @@ def _body_selected_ids(body: Any) -> list[str]:
|
||||
return []
|
||||
|
||||
|
||||
def _body_selected_assistant_ids(body: Any) -> list[str]:
|
||||
for source in (getattr(body, "context", None), getattr(body, "metadata", None)):
|
||||
if isinstance(source, Mapping) and "assistant_knowledge_base_ids" in source:
|
||||
return _as_id_list(source.get("assistant_knowledge_base_ids"))
|
||||
return []
|
||||
|
||||
|
||||
def _assistant_agent_id(body: Any) -> str | None:
|
||||
assistant_id = validate_agent_id(getattr(body, "assistant_id", None))
|
||||
if assistant_id:
|
||||
@@ -106,8 +118,17 @@ def resolve_requested_llmwiki_knowledge_base_ids(body: Any) -> list[str]:
|
||||
return _body_selected_ids(body)
|
||||
|
||||
|
||||
def resolve_requested_assistant_knowledge_base_ids(body: Any) -> list[str]:
|
||||
"""Return assistant knowledge bases explicitly selected for the run."""
|
||||
|
||||
return _body_selected_assistant_ids(body)
|
||||
|
||||
|
||||
def llmwiki_selection_requested(body: Any) -> bool:
|
||||
return bool(resolve_requested_llmwiki_knowledge_base_ids(body))
|
||||
return bool(
|
||||
resolve_requested_llmwiki_knowledge_base_ids(body)
|
||||
or resolve_requested_assistant_knowledge_base_ids(body)
|
||||
)
|
||||
|
||||
|
||||
def _latest_human_query(graph_input: dict[str, Any]) -> str:
|
||||
@@ -149,13 +170,14 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
|
||||
if result.get("kind") == "wiki_page":
|
||||
sections = list(result.get("matched_sections") or [])
|
||||
content = "\n\n".join(str(section.get("content") or "") for section in sections if section.get("content"))
|
||||
content = content or str(result.get("content") or result.get("content_markdown") or "")
|
||||
raw = {**result, "source": mapping.get("name") or "LLMWiki"}
|
||||
return {
|
||||
"index": index,
|
||||
"id": result.get("wiki_page_id") or result.get("id"),
|
||||
"title": result.get("title") or mapping.get("name") or f"Wiki 页面 {index}",
|
||||
"type": "llmwiki_wiki",
|
||||
"snippet": _truncate(content or str(result.get("summary") or ""), 360),
|
||||
"snippet": _truncate(content, 360),
|
||||
"content": content,
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"score": float(result.get("score") or 0),
|
||||
@@ -191,18 +213,52 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
|
||||
}
|
||||
|
||||
|
||||
def _source_from_assistant_result(result: dict[str, Any], index: int) -> dict[str, Any]:
|
||||
title = str(result.get("title") or f"助手知识库片段 {index}")
|
||||
matched_section = result.get("matched_section") if isinstance(result.get("matched_section"), Mapping) else {}
|
||||
content = str(
|
||||
matched_section.get("content")
|
||||
or result.get("content_markdown")
|
||||
or result.get("content")
|
||||
or result.get("summary")
|
||||
or ""
|
||||
)
|
||||
# The retrieval citation must expose the matched knowledge itself. A page
|
||||
# summary is display metadata and can omit the fact that answered the query.
|
||||
snippet = content
|
||||
raw = {
|
||||
**result,
|
||||
"source": "助手知识库",
|
||||
"url": "/page/workspace/knowledge/assistant",
|
||||
}
|
||||
return {
|
||||
"index": index,
|
||||
"id": result.get("id") or result.get("slug") or f"assistant:{index}",
|
||||
"title": title,
|
||||
"type": "assistant_knowledge",
|
||||
"snippet": _truncate(snippet, 360),
|
||||
"content": content,
|
||||
"source": "助手知识库",
|
||||
"score": float(result.get("score") or 0),
|
||||
"url": "/page/workspace/knowledge/assistant",
|
||||
"skillName": "assistant_knowledge_search",
|
||||
"mode": "citation",
|
||||
"raw": raw,
|
||||
}
|
||||
|
||||
|
||||
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
if not sources:
|
||||
return "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
return "当前用户选择了知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
|
||||
lines = [
|
||||
"当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||||
"当前用户选择了知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||||
"请优先依据这些资料回答;如果资料不足,请明确说明不足之处,不要编造。",
|
||||
"引用规则:正文关键结论必须使用 [1]、[2] 这样的编号引用下方资料;只能引用存在的编号;不要在正文末尾重复输出完整参考文献列表,前端会展示引用卡片。",
|
||||
"",
|
||||
f"用户问题:{query}",
|
||||
"",
|
||||
f'<llmwiki_wiki_reference_batch source_count="{len(sources)}">',
|
||||
f'<knowledge_reference_batch source_count="{len(sources)}">',
|
||||
]
|
||||
for source in sources:
|
||||
meta = " / ".join(
|
||||
@@ -213,12 +269,12 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
)
|
||||
if item
|
||||
)
|
||||
lines.append(f"[{source['index']}] {source.get('title') or 'LLMWiki 片段'}")
|
||||
lines.append(f"[{source['index']}] {source.get('title') or '知识库片段'}")
|
||||
if meta:
|
||||
lines.append(f"来源:{meta}")
|
||||
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
|
||||
lines.append("")
|
||||
lines.append("</llmwiki_wiki_reference_batch>")
|
||||
lines.append("</knowledge_reference_batch>")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
@@ -229,52 +285,102 @@ async def build_llmwiki_rag_context(
|
||||
graph_input: dict[str, Any],
|
||||
) -> LlmWikiRagContext:
|
||||
selected_ids = resolve_requested_llmwiki_knowledge_base_ids(body)
|
||||
if not selected_ids:
|
||||
return LlmWikiRagContext([], 0, "", [], "")
|
||||
|
||||
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
|
||||
local_config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
store = _store(request)
|
||||
if store is None:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
assistant_selected_ids = resolve_requested_assistant_knowledge_base_ids(body)
|
||||
if not selected_ids and not assistant_selected_ids:
|
||||
return LlmWikiRagContext([], [], 0, "", [], "")
|
||||
|
||||
query = _latest_human_query(graph_input)
|
||||
if not query:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], "")
|
||||
|
||||
user_id, is_admin = await _actor(request)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for mapping_id in selected_ids:
|
||||
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
|
||||
rows.append(row)
|
||||
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if search_service is None:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
0,
|
||||
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
try:
|
||||
payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
|
||||
except WikiIndexError as exc:
|
||||
logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
|
||||
return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。", [], query)
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
sources: list[dict[str, Any]] = []
|
||||
for result in payload.get("results") or []:
|
||||
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
sources.append(_source_from_result(result, mapping, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
return LlmWikiRagContext(selected_ids, len(sources), _build_prompt(query, sources), sources, query)
|
||||
|
||||
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
|
||||
local_config = request.app.state.config.llmwiki.local_wiki_index
|
||||
user_id, is_admin = await _actor(request)
|
||||
|
||||
if selected_ids:
|
||||
llmwiki_available = (local_config.enabled and local_config.internal_search_enabled) or runtime.weknora_enabled
|
||||
store = _store(request)
|
||||
if not llmwiki_available or store is None:
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], query)
|
||||
else:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for mapping_id in selected_ids:
|
||||
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
|
||||
rows.append(row)
|
||||
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if search_service is None:
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
0,
|
||||
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
try:
|
||||
payload = (
|
||||
await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
|
||||
if search_service is not None
|
||||
else {"results": []}
|
||||
)
|
||||
except WikiIndexError as exc:
|
||||
logger.warning(
|
||||
"Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s",
|
||||
sanitize_log_param(user_id),
|
||||
len(rows),
|
||||
exc.code,
|
||||
)
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
0,
|
||||
f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
payload = {"results": []}
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
for result in payload.get("results") or []:
|
||||
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
source = _source_from_result(result, mapping, len(sources) + 1)
|
||||
if not str(source.get("content") or "").strip():
|
||||
continue
|
||||
sources.append(source)
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
|
||||
if assistant_selected_ids and len(sources) < _MAX_SOURCES:
|
||||
assistant_store = _assistant_store(request)
|
||||
if assistant_store is not None:
|
||||
rows = await assistant_store.search_pages(
|
||||
base_ids=assistant_selected_ids,
|
||||
query=query,
|
||||
limit=_MAX_SOURCES - len(sources),
|
||||
include_chunks=True,
|
||||
embedding_client=getattr(request.app.state, "llmwiki_embedding", None),
|
||||
)
|
||||
for row in rows:
|
||||
sources.append(_source_from_assistant_result(row, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
len(sources),
|
||||
_build_prompt(query, sources),
|
||||
sources,
|
||||
query,
|
||||
)
|
||||
|
||||
|
||||
def attach_llmwiki_rag_to_graph_input(
|
||||
@@ -294,6 +400,7 @@ def attach_llmwiki_rag_to_graph_input(
|
||||
existing = str(extra.get("prompt_prefix") or "").strip()
|
||||
extra["prompt_prefix"] = f"{existing}\n\n{rag.prompt}".strip() if existing else rag.prompt
|
||||
extra["llmwiki_knowledge_base_ids"] = rag.knowledge_base_ids
|
||||
extra["assistant_knowledge_base_ids"] = rag.assistant_knowledge_base_ids
|
||||
extra["llmwiki_reference_query"] = rag.query
|
||||
if rag.sources:
|
||||
extra["llmwiki_reference_sources"] = rag.sources
|
||||
|
||||
@@ -461,7 +461,7 @@ async def _set_analytics_read_timeout(session) -> None:
|
||||
- MariaDB: ``max_statement_time`` (seconds)
|
||||
- MySQL <= 5.6: no per-statement timeout variable — skip it entirely
|
||||
(5.6 raises "Unknown system variable" otherwise)
|
||||
- PostgreSQL: ``statement_timeout`` (milliseconds)
|
||||
- PostgreSQL/openGauss: ``statement_timeout`` (milliseconds)
|
||||
"""
|
||||
conn = await session.connection()
|
||||
dialect = conn.dialect
|
||||
@@ -474,7 +474,7 @@ async def _set_analytics_read_timeout(session) -> None:
|
||||
await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
|
||||
# else: MySQL <= 5.6 has no statement-timeout variable — skip silently.
|
||||
await _try_set_session_value(session, f"SET SESSION innodb_lock_wait_timeout = {_LEADERBOARD_LOCK_WAIT_SECONDS}")
|
||||
elif name == "postgresql":
|
||||
elif name in {"postgresql", "opengauss"}:
|
||||
await _try_set_session_value(session, f"SET LOCAL statement_timeout = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
|
||||
|
||||
|
||||
|
||||
@@ -1,329 +1,625 @@
|
||||
import logging
|
||||
import mimetypes
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import FileResponse, PlainTextResponse, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.path_utils import aresolve_thread_virtual_path
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["artifacts"])
|
||||
|
||||
ACTIVE_CONTENT_MIME_TYPES = {
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
"image/svg+xml",
|
||||
}
|
||||
|
||||
EDITABLE_TEXT_SUFFIXES = {
|
||||
".md",
|
||||
".markdown",
|
||||
".txt",
|
||||
".json",
|
||||
".yaml",
|
||||
".yml",
|
||||
".csv",
|
||||
".xml",
|
||||
".log",
|
||||
}
|
||||
|
||||
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
|
||||
|
||||
|
||||
def coerce_artifact_virtual_path(path: str) -> str:
|
||||
"""Normalize an artifacts URL path to a sandbox virtual path.
|
||||
|
||||
Streaming write previews may request a bare filename such as
|
||||
``未命名文件.md`` before the model has emitted a real
|
||||
``/mnt/user-data/outputs/...`` path. Mapping a single path segment into
|
||||
the thread outputs directory turns that into a normal 404 (file not written
|
||||
yet) instead of HTTP 400 ``Path must start with /mnt/user-data``.
|
||||
"""
|
||||
stripped = path.strip().lstrip("/").replace("\\", "/")
|
||||
if not stripped:
|
||||
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
|
||||
if stripped == _VIRTUAL_USER_DATA_PREFIX or stripped.startswith(_VIRTUAL_USER_DATA_PREFIX + "/"):
|
||||
return stripped
|
||||
if "/" in stripped or stripped in {".", ".."}:
|
||||
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
|
||||
return f"{_VIRTUAL_USER_DATA_PREFIX}/outputs/{stripped}"
|
||||
|
||||
|
||||
class ArtifactUpdateRequest(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
def _build_content_disposition(disposition_type: str, filename: str) -> str:
|
||||
"""Build an RFC 5987 encoded Content-Disposition header value."""
|
||||
return f"{disposition_type}; filename*=UTF-8''{quote(filename)}"
|
||||
|
||||
|
||||
def _build_attachment_headers(filename: str, extra_headers: dict[str, str] | None = None) -> dict[str, str]:
|
||||
headers = {"Content-Disposition": _build_content_disposition("attachment", filename)}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
return headers
|
||||
|
||||
|
||||
def is_text_file_by_content(path: Path, sample_size: int = 8192) -> bool:
|
||||
"""Check if file is text by examining content for null bytes."""
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
chunk = f.read(sample_size)
|
||||
# Text files shouldn't contain null bytes
|
||||
return b"\x00" not in chunk
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _extract_file_from_skill_archive(zip_path: Path, internal_path: str) -> bytes | None:
|
||||
"""Extract a file from a .skill ZIP archive.
|
||||
|
||||
Args:
|
||||
zip_path: Path to the .skill file (ZIP archive).
|
||||
internal_path: Path to the file inside the archive (e.g., "SKILL.md").
|
||||
|
||||
Returns:
|
||||
The file content as bytes, or None if not found.
|
||||
"""
|
||||
if not zipfile.is_zipfile(zip_path):
|
||||
return None
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path, "r") as zip_ref:
|
||||
# List all files in the archive
|
||||
namelist = zip_ref.namelist()
|
||||
|
||||
# Try direct path first
|
||||
if internal_path in namelist:
|
||||
return zip_ref.read(internal_path)
|
||||
|
||||
# Try with any top-level directory prefix (e.g., "skill-name/SKILL.md")
|
||||
for name in namelist:
|
||||
if name.endswith("/" + internal_path) or name == internal_path:
|
||||
return zip_ref.read(name)
|
||||
|
||||
# Not found
|
||||
return None
|
||||
except (zipfile.BadZipFile, KeyError):
|
||||
return None
|
||||
|
||||
|
||||
def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
|
||||
"""Walk an outputs directory and return artifact listing entries.
|
||||
|
||||
Each entry has ``path`` (virtual ``mnt/user-data/outputs/...``), ``name``,
|
||||
``size_bytes``, ``mime_type``, and ``modified_at`` (ISO 8601 UTC). Symlinks
|
||||
that escape the outputs root are dropped to keep the listing scoped to the
|
||||
thread. The result is sorted by ``modified_at`` descending (newest first).
|
||||
"""
|
||||
if not outputs_dir.exists() or not outputs_dir.is_dir():
|
||||
return []
|
||||
|
||||
outputs_root = outputs_dir.resolve()
|
||||
files: list[dict] = []
|
||||
for entry in outputs_root.rglob("*"):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
try:
|
||||
resolved = entry.resolve()
|
||||
resolved.relative_to(outputs_root)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
relative = resolved.relative_to(outputs_root).as_posix()
|
||||
mime_type, _ = mimetypes.guess_type(resolved.name)
|
||||
stat = resolved.stat()
|
||||
files.append(
|
||||
{
|
||||
"path": f"mnt/user-data/outputs/{relative}",
|
||||
"name": resolved.name,
|
||||
"size_bytes": stat.st_size,
|
||||
"mime_type": mime_type,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
|
||||
}
|
||||
)
|
||||
|
||||
files.sort(key=lambda f: f["modified_at"], reverse=True)
|
||||
return files
|
||||
|
||||
|
||||
@router.get(
|
||||
"/threads/{thread_id}/artifacts",
|
||||
summary="List Artifact Files",
|
||||
description="List all files under the thread's outputs directory (/mnt/user-data/outputs/). Returns an empty list if the thread has no outputs.",
|
||||
)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def list_artifacts(thread_id: str, request: Request) -> dict:
|
||||
"""列出 thread 在 ``outputs/`` 下生成的产物文件。
|
||||
|
||||
圆桌 Step 3(``HighFidelityReport``)对每个**子席位** thread_id 调用本接口,
|
||||
展示各智能体写入沙箱 ``mnt/user-data/outputs/`` 的文件;协调智能体 thread
|
||||
通常无独立交付文件。
|
||||
|
||||
遍历路径::
|
||||
|
||||
.deer-flow/users/{user_id}/threads/{thread_id}/user-data/outputs/
|
||||
|
||||
返回 ``{ "thread_id", "files": [{ path, name, size_bytes, mime_type, modified_at }] }``,
|
||||
目录不存在时 ``files`` 为空数组(非 404)。
|
||||
"""
|
||||
outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
|
||||
return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/threads/{thread_id}/artifacts/{path:path}",
|
||||
summary="Get Artifact File",
|
||||
description="Retrieve an artifact file generated by the AI agent. Text and binary files can be viewed inline, while active web content is always downloaded.",
|
||||
)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_artifact(thread_id: str, path: str, request: Request, download: bool = False) -> Response:
|
||||
"""Get an artifact file by its path.
|
||||
|
||||
The endpoint automatically detects file types and returns appropriate content types.
|
||||
Use the `download` query parameter to force file download for non-active content.
|
||||
|
||||
Args:
|
||||
thread_id: The thread ID.
|
||||
path: The artifact path with virtual prefix (e.g., mnt/user-data/outputs/file.txt).
|
||||
request: FastAPI request object (automatically injected).
|
||||
|
||||
Returns:
|
||||
The file content as a FileResponse with appropriate content type:
|
||||
- Active content (HTML/XHTML/SVG): Served as download attachment
|
||||
- Text files: Plain text with proper MIME type
|
||||
- Binary files: Inline display with download option
|
||||
|
||||
Raises:
|
||||
HTTPException:
|
||||
- 400 if path is invalid or not a file
|
||||
- 403 if access denied (path traversal detected)
|
||||
- 404 if file not found
|
||||
|
||||
Query Parameters:
|
||||
download (bool): If true, forces attachment download for file types that are
|
||||
otherwise returned inline or as plain text. Active HTML/XHTML/SVG content
|
||||
is always downloaded regardless of this flag.
|
||||
|
||||
Example:
|
||||
- Get text file inline: `/api/threads/abc123/artifacts/mnt/user-data/outputs/notes.txt`
|
||||
- Download file: `/api/threads/abc123/artifacts/mnt/user-data/outputs/data.csv?download=true`
|
||||
- Active web content such as `.html`, `.xhtml`, and `.svg` artifacts is always downloaded
|
||||
"""
|
||||
path = coerce_artifact_virtual_path(path)
|
||||
# Check if this is a request for a file inside a .skill archive (e.g., xxx.skill/SKILL.md)
|
||||
if ".skill/" in path:
|
||||
# Split the path at ".skill/" to get the ZIP file path and internal path
|
||||
skill_marker = ".skill/"
|
||||
marker_pos = path.find(skill_marker)
|
||||
skill_file_path = path[: marker_pos + len(".skill")] # e.g., "mnt/user-data/outputs/my-skill.skill"
|
||||
internal_path = path[marker_pos + len(skill_marker) :] # e.g., "SKILL.md"
|
||||
|
||||
actual_skill_path = await aresolve_thread_virtual_path(thread_id, skill_file_path)
|
||||
|
||||
if not actual_skill_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Skill file not found: {skill_file_path}")
|
||||
|
||||
if not actual_skill_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {skill_file_path}")
|
||||
|
||||
# Extract the file from the .skill archive
|
||||
content = _extract_file_from_skill_archive(actual_skill_path, internal_path)
|
||||
if content is None:
|
||||
raise HTTPException(status_code=404, detail=f"File '{internal_path}' not found in skill archive")
|
||||
|
||||
# Determine MIME type based on the internal file
|
||||
mime_type, _ = mimetypes.guess_type(internal_path)
|
||||
# Add cache headers to avoid repeated ZIP extraction (cache for 5 minutes)
|
||||
cache_headers = {"Cache-Control": "private, max-age=300"}
|
||||
download_name = Path(internal_path).name or actual_skill_path.stem
|
||||
if download or mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=_build_attachment_headers(download_name, cache_headers))
|
||||
|
||||
if mime_type and mime_type.startswith("text/"):
|
||||
return PlainTextResponse(content=content.decode("utf-8"), media_type=mime_type, headers=cache_headers)
|
||||
|
||||
# Default to plain text for unknown types that look like text
|
||||
try:
|
||||
return PlainTextResponse(content=content.decode("utf-8"), media_type="text/plain", headers=cache_headers)
|
||||
except UnicodeDecodeError:
|
||||
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=cache_headers)
|
||||
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, path)
|
||||
|
||||
logger.info(f"Resolving artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
|
||||
|
||||
if not actual_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
|
||||
|
||||
if not actual_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(actual_path)
|
||||
|
||||
if download:
|
||||
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
|
||||
|
||||
# Always force download for active content types to prevent script execution
|
||||
# in the application origin when users open generated artifacts.
|
||||
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
|
||||
|
||||
if mime_type and mime_type.startswith("text/"):
|
||||
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
|
||||
|
||||
if is_text_file_by_content(actual_path):
|
||||
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
|
||||
|
||||
return Response(content=actual_path.read_bytes(), media_type=mime_type, headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)})
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/threads/{thread_id}/artifacts/{path:path}",
|
||||
summary="Update Artifact File",
|
||||
description="Update a text artifact file in the current thread workspace.",
|
||||
)
|
||||
@require_permission("threads", "write", owner_check=True)
|
||||
async def update_artifact(
|
||||
thread_id: str,
|
||||
path: str,
|
||||
request: Request,
|
||||
body: ArtifactUpdateRequest,
|
||||
) -> Response:
|
||||
"""Update a text artifact.
|
||||
|
||||
This endpoint is intentionally limited to normal text artifacts. Files inside
|
||||
.skill archives and active web content remain read-only from the artifact UI.
|
||||
"""
|
||||
if ".skill/" in path:
|
||||
raise HTTPException(status_code=400, detail="Skill archive contents are read-only.")
|
||||
|
||||
path = coerce_artifact_virtual_path(path)
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, path)
|
||||
logger.info(f"Updating artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
|
||||
|
||||
if not actual_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
|
||||
|
||||
if not actual_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(actual_path)
|
||||
suffix = actual_path.suffix.lower()
|
||||
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
raise HTTPException(status_code=400, detail="Active web artifacts are read-only.")
|
||||
|
||||
if suffix not in EDITABLE_TEXT_SUFFIXES and not (mime_type and mime_type.startswith("text/")):
|
||||
raise HTTPException(status_code=400, detail="Only text artifacts can be edited.")
|
||||
|
||||
actual_path.write_text(body.content, encoding="utf-8")
|
||||
return PlainTextResponse(content=body.content, media_type=mime_type or "text/plain")
|
||||
import asyncio
|
||||
import logging
|
||||
import mimetypes
|
||||
import time
|
||||
import zipfile
|
||||
from collections import OrderedDict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, PlainTextResponse, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.path_utils import aresolve_thread_virtual_path
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["artifacts"])
|
||||
|
||||
ACTIVE_CONTENT_MIME_TYPES = {
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
"image/svg+xml",
|
||||
}
|
||||
|
||||
EDITABLE_TEXT_SUFFIXES = {
|
||||
".md",
|
||||
".markdown",
|
||||
".txt",
|
||||
".json",
|
||||
".yaml",
|
||||
".yml",
|
||||
".csv",
|
||||
".xml",
|
||||
".log",
|
||||
}
|
||||
|
||||
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
|
||||
|
||||
ArtifactLibraryKind = Literal["all", "document", "spreadsheet", "text_data", "archive", "other"]
|
||||
ArtifactLibrarySource = Literal["all", "general_qa", "agent"]
|
||||
|
||||
_MEDIA_MIME_PREFIXES = ("image/", "audio/", "video/")
|
||||
_MEDIA_SUFFIXES = {
|
||||
".aac",
|
||||
".aiff",
|
||||
".ape",
|
||||
".avi",
|
||||
".bmp",
|
||||
".flac",
|
||||
".gif",
|
||||
".heic",
|
||||
".ico",
|
||||
".jpeg",
|
||||
".jpg",
|
||||
".m4a",
|
||||
".m4v",
|
||||
".mkv",
|
||||
".mov",
|
||||
".mp3",
|
||||
".mp4",
|
||||
".ogg",
|
||||
".png",
|
||||
".svg",
|
||||
".tiff",
|
||||
".wav",
|
||||
".webm",
|
||||
".webp",
|
||||
".wma",
|
||||
}
|
||||
_DOCUMENT_SUFFIXES = {
|
||||
".doc",
|
||||
".docx",
|
||||
".epub",
|
||||
".odt",
|
||||
".pdf",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".rtf",
|
||||
}
|
||||
_SPREADSHEET_SUFFIXES = {".csv", ".ods", ".tsv", ".xls", ".xlsx"}
|
||||
_TEXT_DATA_SUFFIXES = {
|
||||
".bash",
|
||||
".c",
|
||||
".conf",
|
||||
".cpp",
|
||||
".css",
|
||||
".go",
|
||||
".h",
|
||||
".html",
|
||||
".ini",
|
||||
".java",
|
||||
".js",
|
||||
".json",
|
||||
".jsonl",
|
||||
".log",
|
||||
".markdown",
|
||||
".md",
|
||||
".mmd",
|
||||
".py",
|
||||
".rs",
|
||||
".sh",
|
||||
".sql",
|
||||
".toml",
|
||||
".ts",
|
||||
".tsx",
|
||||
".txt",
|
||||
".xml",
|
||||
".yaml",
|
||||
".yml",
|
||||
}
|
||||
_ARCHIVE_SUFFIXES = {".7z", ".gz", ".rar", ".skill", ".tar", ".tgz", ".zip"}
|
||||
_ARTIFACT_LIBRARY_CACHE_TTL_SECONDS = 15.0
|
||||
_ARTIFACT_LIBRARY_CACHE_MAX_USERS = 128
|
||||
_ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 8
|
||||
_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE = 500
|
||||
_artifact_library_cache: OrderedDict[str, tuple[float, list[dict]]] = OrderedDict()
|
||||
|
||||
|
||||
def coerce_artifact_virtual_path(path: str) -> str:
|
||||
"""Normalize an artifacts URL path to a sandbox virtual path.
|
||||
|
||||
Streaming write previews may request a bare filename such as
|
||||
``未命名文件.md`` before the model has emitted a real
|
||||
``/mnt/user-data/outputs/...`` path. Mapping a single path segment into
|
||||
the thread outputs directory turns that into a normal 404 (file not written
|
||||
yet) instead of HTTP 400 ``Path must start with /mnt/user-data``.
|
||||
"""
|
||||
stripped = path.strip().lstrip("/").replace("\\", "/")
|
||||
if not stripped:
|
||||
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
|
||||
if stripped == _VIRTUAL_USER_DATA_PREFIX or stripped.startswith(_VIRTUAL_USER_DATA_PREFIX + "/"):
|
||||
return stripped
|
||||
if "/" in stripped or stripped in {".", ".."}:
|
||||
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
|
||||
return f"{_VIRTUAL_USER_DATA_PREFIX}/outputs/{stripped}"
|
||||
|
||||
|
||||
class ArtifactUpdateRequest(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
class ArtifactLibraryItem(BaseModel):
|
||||
thread_id: str
|
||||
thread_title: str
|
||||
agent_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
thread_type: str | None = None
|
||||
source_type: Literal["general_qa", "agent"]
|
||||
path: str
|
||||
name: str
|
||||
size_bytes: int
|
||||
mime_type: str | None = None
|
||||
modified_at: str
|
||||
file_kind: str
|
||||
|
||||
|
||||
class ArtifactLibraryResponse(BaseModel):
|
||||
total: int
|
||||
offset: int
|
||||
limit: int
|
||||
items: list[ArtifactLibraryItem]
|
||||
|
||||
|
||||
def _build_content_disposition(disposition_type: str, filename: str) -> str:
|
||||
"""Build an RFC 5987 encoded Content-Disposition header value."""
|
||||
return f"{disposition_type}; filename*=UTF-8''{quote(filename)}"
|
||||
|
||||
|
||||
def _build_attachment_headers(filename: str, extra_headers: dict[str, str] | None = None) -> dict[str, str]:
|
||||
headers = {"Content-Disposition": _build_content_disposition("attachment", filename)}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
return headers
|
||||
|
||||
|
||||
def is_text_file_by_content(path: Path, sample_size: int = 8192) -> bool:
|
||||
"""Check if file is text by examining content for null bytes."""
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
chunk = f.read(sample_size)
|
||||
# Text files shouldn't contain null bytes
|
||||
return b"\x00" not in chunk
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _extract_file_from_skill_archive(zip_path: Path, internal_path: str) -> bytes | None:
|
||||
"""Extract a file from a .skill ZIP archive.
|
||||
|
||||
Args:
|
||||
zip_path: Path to the .skill file (ZIP archive).
|
||||
internal_path: Path to the file inside the archive (e.g., "SKILL.md").
|
||||
|
||||
Returns:
|
||||
The file content as bytes, or None if not found.
|
||||
"""
|
||||
if not zipfile.is_zipfile(zip_path):
|
||||
return None
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path, "r") as zip_ref:
|
||||
# List all files in the archive
|
||||
namelist = zip_ref.namelist()
|
||||
|
||||
# Try direct path first
|
||||
if internal_path in namelist:
|
||||
return zip_ref.read(internal_path)
|
||||
|
||||
# Try with any top-level directory prefix (e.g., "skill-name/SKILL.md")
|
||||
for name in namelist:
|
||||
if name.endswith("/" + internal_path) or name == internal_path:
|
||||
return zip_ref.read(name)
|
||||
|
||||
# Not found
|
||||
return None
|
||||
except (zipfile.BadZipFile, KeyError):
|
||||
return None
|
||||
|
||||
|
||||
def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
|
||||
"""Walk an outputs directory and return artifact listing entries.
|
||||
|
||||
Each entry has ``path`` (virtual ``mnt/user-data/outputs/...``), ``name``,
|
||||
``size_bytes``, ``mime_type``, and ``modified_at`` (ISO 8601 UTC). Symlinks
|
||||
that escape the outputs root are dropped to keep the listing scoped to the
|
||||
thread. The result is sorted by ``modified_at`` descending (newest first).
|
||||
"""
|
||||
if not outputs_dir.exists() or not outputs_dir.is_dir():
|
||||
return []
|
||||
|
||||
outputs_root = outputs_dir.resolve()
|
||||
files: list[dict] = []
|
||||
for entry in outputs_root.rglob("*"):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
try:
|
||||
resolved = entry.resolve()
|
||||
resolved.relative_to(outputs_root)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
try:
|
||||
relative = resolved.relative_to(outputs_root).as_posix()
|
||||
mime_type, _ = mimetypes.guess_type(resolved.name)
|
||||
stat = resolved.stat()
|
||||
except OSError:
|
||||
# A run may atomically replace or remove a file while this directory
|
||||
# is being scanned. Treat that transient entry as absent instead of
|
||||
# failing the complete thread/library listing.
|
||||
continue
|
||||
files.append(
|
||||
{
|
||||
"path": f"mnt/user-data/outputs/{relative}",
|
||||
"name": resolved.name,
|
||||
"size_bytes": stat.st_size,
|
||||
"mime_type": mime_type,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(),
|
||||
}
|
||||
)
|
||||
|
||||
files.sort(key=lambda f: f["modified_at"], reverse=True)
|
||||
return files
|
||||
|
||||
|
||||
def artifact_library_kind(name: str) -> str:
|
||||
"""Classify one non-media artifact for the library filters."""
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in _DOCUMENT_SUFFIXES:
|
||||
return "document"
|
||||
if suffix in _SPREADSHEET_SUFFIXES:
|
||||
return "spreadsheet"
|
||||
if suffix in _TEXT_DATA_SUFFIXES:
|
||||
return "text_data"
|
||||
if suffix in _ARCHIVE_SUFFIXES:
|
||||
return "archive"
|
||||
return "other"
|
||||
|
||||
|
||||
def is_artifact_library_file(file: dict) -> bool:
|
||||
"""Return whether a generated file belongs in the document-only library."""
|
||||
mime_type = str(file.get("mime_type") or "").lower()
|
||||
if mime_type.startswith(_MEDIA_MIME_PREFIXES):
|
||||
return False
|
||||
return Path(str(file.get("name") or file.get("path") or "")).suffix.lower() not in _MEDIA_SUFFIXES
|
||||
|
||||
|
||||
async def _search_artifact_library_threads(thread_store) -> list[dict]:
|
||||
"""Load every normal conversation owned by the current request user."""
|
||||
rows: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
offset = 0
|
||||
while True:
|
||||
batch = await thread_store.search(
|
||||
limit=_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE,
|
||||
offset=offset,
|
||||
exclude_system=True,
|
||||
)
|
||||
if not batch:
|
||||
break
|
||||
added = 0
|
||||
for row in batch:
|
||||
thread_id = str(row.get("thread_id") or "")
|
||||
if not thread_id or thread_id in seen:
|
||||
continue
|
||||
seen.add(thread_id)
|
||||
rows.append(row)
|
||||
added += 1
|
||||
if len(batch) < _ARTIFACT_LIBRARY_THREAD_BATCH_SIZE or added == 0:
|
||||
break
|
||||
offset += len(batch)
|
||||
return rows
|
||||
|
||||
|
||||
async def _scan_artifact_library(thread_store) -> list[dict]:
|
||||
"""Scan generated outputs for all current-user conversation threads."""
|
||||
rows = await _search_artifact_library_threads(thread_store)
|
||||
paths = get_paths()
|
||||
semaphore = asyncio.Semaphore(_ARTIFACT_LIBRARY_SCAN_CONCURRENCY)
|
||||
|
||||
async def scan_thread(row: dict) -> list[dict]:
|
||||
thread_id = str(row.get("thread_id") or "")
|
||||
if not thread_id:
|
||||
return []
|
||||
async with semaphore:
|
||||
try:
|
||||
path_user_id = await aresolve_path_user_id(thread_id)
|
||||
outputs_dir = paths.sandbox_outputs_dir(thread_id, user_id=path_user_id)
|
||||
files = await asyncio.to_thread(collect_outputs_listing, outputs_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to scan artifacts for library thread %s", thread_id, exc_info=True)
|
||||
return []
|
||||
|
||||
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
|
||||
agent_id = metadata.get("agent_id")
|
||||
agent_name = metadata.get("agent_name")
|
||||
thread_type = metadata.get("thread_type")
|
||||
normalized_agent_id = agent_id.strip() if isinstance(agent_id, str) else ""
|
||||
normalized_agent_name = agent_name.strip() if isinstance(agent_name, str) else ""
|
||||
title = str(row.get("display_name") or "新对话")
|
||||
items: list[dict] = []
|
||||
for file in files:
|
||||
if not is_artifact_library_file(file):
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"thread_title": title,
|
||||
"agent_id": normalized_agent_id or None,
|
||||
"agent_name": normalized_agent_name or None,
|
||||
"thread_type": thread_type if isinstance(thread_type, str) and thread_type else None,
|
||||
"source_type": "agent" if normalized_agent_id else "general_qa",
|
||||
**file,
|
||||
"file_kind": artifact_library_kind(str(file.get("name") or "")),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
nested = await asyncio.gather(*(scan_thread(row) for row in rows))
|
||||
items = [item for group in nested for item in group]
|
||||
items.sort(
|
||||
key=lambda item: (
|
||||
str(item.get("modified_at") or ""),
|
||||
str(item.get("thread_id") or ""),
|
||||
str(item.get("path") or ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
async def _cached_artifact_library(thread_store, *, refresh: bool) -> list[dict]:
|
||||
user_id = str(get_effective_user_id() or "default")
|
||||
now = time.monotonic()
|
||||
cached = _artifact_library_cache.get(user_id)
|
||||
if not refresh and cached and cached[0] > now:
|
||||
_artifact_library_cache.move_to_end(user_id)
|
||||
return cached[1]
|
||||
|
||||
items = await _scan_artifact_library(thread_store)
|
||||
_artifact_library_cache[user_id] = (now + _ARTIFACT_LIBRARY_CACHE_TTL_SECONDS, items)
|
||||
_artifact_library_cache.move_to_end(user_id)
|
||||
while len(_artifact_library_cache) > _ARTIFACT_LIBRARY_CACHE_MAX_USERS:
|
||||
_artifact_library_cache.popitem(last=False)
|
||||
return items
|
||||
|
||||
|
||||
@router.get(
|
||||
"/artifact-library",
|
||||
response_model=ArtifactLibraryResponse,
|
||||
summary="List Generated Files Across Conversations",
|
||||
description="Aggregate non-media files generated in the current user's normal conversations.",
|
||||
)
|
||||
@require_permission("threads", "read")
|
||||
async def list_artifact_library(
|
||||
request: Request,
|
||||
query: str | None = Query(default=None, max_length=200),
|
||||
file_kind: ArtifactLibraryKind = Query(default="all"),
|
||||
source_type: ArtifactLibrarySource = Query(default="all"),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
refresh: bool = Query(default=False),
|
||||
) -> ArtifactLibraryResponse:
|
||||
"""Return a flat, newest-first library of generated conversation files.
|
||||
|
||||
The filesystem remains the source of truth: every owned, non-system thread's
|
||||
``outputs`` directory is scanned, including files written without an
|
||||
explicit ``present_files`` call. Uploads and image/audio/video outputs are
|
||||
intentionally outside this collection.
|
||||
"""
|
||||
from app.gateway.deps import get_agent_store, get_thread_store
|
||||
|
||||
items = await _cached_artifact_library(get_thread_store(request), refresh=refresh)
|
||||
agent_ids = list(dict.fromkeys(str(item.get("agent_id") or "") for item in items if item.get("agent_id")))
|
||||
agent_names: dict[str, str] = {}
|
||||
if agent_ids:
|
||||
try:
|
||||
agents = await get_agent_store(request).list_by_ids(
|
||||
agent_ids,
|
||||
str(get_effective_user_id() or "default"),
|
||||
)
|
||||
agent_names = {str(agent.get("id") or ""): str(agent.get("name") or "").strip() for agent in agents if agent.get("id") and agent.get("name")}
|
||||
except Exception:
|
||||
logger.warning("Failed to resolve artifact library agent names", exc_info=True)
|
||||
|
||||
items_with_sources = [
|
||||
{
|
||||
**item,
|
||||
"agent_name": (agent_names.get(str(item.get("agent_id") or "")) or item.get("agent_name") or item.get("agent_id")),
|
||||
}
|
||||
for item in items
|
||||
]
|
||||
normalized_query = (query or "").strip().casefold()
|
||||
filtered = [
|
||||
item
|
||||
for item in items_with_sources
|
||||
if (file_kind == "all" or item["file_kind"] == file_kind)
|
||||
and (source_type == "all" or item["source_type"] == source_type)
|
||||
and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold() or normalized_query in str(item.get("agent_name") or "").casefold())
|
||||
]
|
||||
page = filtered[offset : offset + limit]
|
||||
return ArtifactLibraryResponse(
|
||||
total=len(filtered),
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
items=[ArtifactLibraryItem.model_validate(item) for item in page],
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/threads/{thread_id}/artifacts",
|
||||
summary="List Artifact Files",
|
||||
description="List all files under the thread's outputs directory (/mnt/user-data/outputs/). Returns an empty list if the thread has no outputs.",
|
||||
)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def list_artifacts(thread_id: str, request: Request) -> dict:
|
||||
"""列出 thread 在 ``outputs/`` 下生成的产物文件。
|
||||
|
||||
圆桌 Step 3(``HighFidelityReport``)对每个**子席位** thread_id 调用本接口,
|
||||
展示各智能体写入沙箱 ``mnt/user-data/outputs/`` 的文件;协调智能体 thread
|
||||
通常无独立交付文件。
|
||||
|
||||
遍历路径::
|
||||
|
||||
.deer-flow/users/{user_id}/threads/{thread_id}/user-data/outputs/
|
||||
|
||||
返回 ``{ "thread_id", "files": [{ path, name, size_bytes, mime_type, modified_at }] }``,
|
||||
目录不存在时 ``files`` 为空数组(非 404)。
|
||||
"""
|
||||
outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
|
||||
return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/threads/{thread_id}/artifacts/{path:path}",
|
||||
summary="Get Artifact File",
|
||||
description="Retrieve an artifact file generated by the AI agent. Text and binary files can be viewed inline, while active web content is always downloaded.",
|
||||
)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_artifact(thread_id: str, path: str, request: Request, download: bool = False) -> Response:
|
||||
"""Get an artifact file by its path.
|
||||
|
||||
The endpoint automatically detects file types and returns appropriate content types.
|
||||
Use the `download` query parameter to force file download for non-active content.
|
||||
|
||||
Args:
|
||||
thread_id: The thread ID.
|
||||
path: The artifact path with virtual prefix (e.g., mnt/user-data/outputs/file.txt).
|
||||
request: FastAPI request object (automatically injected).
|
||||
|
||||
Returns:
|
||||
The file content as a FileResponse with appropriate content type:
|
||||
- Active content (HTML/XHTML/SVG): Served as download attachment
|
||||
- Text files: Plain text with proper MIME type
|
||||
- Binary files: Inline display with download option
|
||||
|
||||
Raises:
|
||||
HTTPException:
|
||||
- 400 if path is invalid or not a file
|
||||
- 403 if access denied (path traversal detected)
|
||||
- 404 if file not found
|
||||
|
||||
Query Parameters:
|
||||
download (bool): If true, forces attachment download for file types that are
|
||||
otherwise returned inline or as plain text. Active HTML/XHTML/SVG content
|
||||
is always downloaded regardless of this flag.
|
||||
|
||||
Example:
|
||||
- Get text file inline: `/api/threads/abc123/artifacts/mnt/user-data/outputs/notes.txt`
|
||||
- Download file: `/api/threads/abc123/artifacts/mnt/user-data/outputs/data.csv?download=true`
|
||||
- Active web content such as `.html`, `.xhtml`, and `.svg` artifacts is always downloaded
|
||||
"""
|
||||
path = coerce_artifact_virtual_path(path)
|
||||
# Check if this is a request for a file inside a .skill archive (e.g., xxx.skill/SKILL.md)
|
||||
if ".skill/" in path:
|
||||
# Split the path at ".skill/" to get the ZIP file path and internal path
|
||||
skill_marker = ".skill/"
|
||||
marker_pos = path.find(skill_marker)
|
||||
skill_file_path = path[: marker_pos + len(".skill")] # e.g., "mnt/user-data/outputs/my-skill.skill"
|
||||
internal_path = path[marker_pos + len(skill_marker) :] # e.g., "SKILL.md"
|
||||
|
||||
actual_skill_path = await aresolve_thread_virtual_path(thread_id, skill_file_path)
|
||||
|
||||
if not actual_skill_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Skill file not found: {skill_file_path}")
|
||||
|
||||
if not actual_skill_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {skill_file_path}")
|
||||
|
||||
# Extract the file from the .skill archive
|
||||
content = _extract_file_from_skill_archive(actual_skill_path, internal_path)
|
||||
if content is None:
|
||||
raise HTTPException(status_code=404, detail=f"File '{internal_path}' not found in skill archive")
|
||||
|
||||
# Determine MIME type based on the internal file
|
||||
mime_type, _ = mimetypes.guess_type(internal_path)
|
||||
# Add cache headers to avoid repeated ZIP extraction (cache for 5 minutes)
|
||||
cache_headers = {"Cache-Control": "private, max-age=300"}
|
||||
download_name = Path(internal_path).name or actual_skill_path.stem
|
||||
if download or mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=_build_attachment_headers(download_name, cache_headers))
|
||||
|
||||
if mime_type and mime_type.startswith("text/"):
|
||||
return PlainTextResponse(content=content.decode("utf-8"), media_type=mime_type, headers=cache_headers)
|
||||
|
||||
# Default to plain text for unknown types that look like text
|
||||
try:
|
||||
return PlainTextResponse(content=content.decode("utf-8"), media_type="text/plain", headers=cache_headers)
|
||||
except UnicodeDecodeError:
|
||||
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=cache_headers)
|
||||
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, path)
|
||||
|
||||
logger.info(f"Resolving artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
|
||||
|
||||
if not actual_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
|
||||
|
||||
if not actual_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(actual_path)
|
||||
|
||||
if download:
|
||||
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
|
||||
|
||||
# Always force download for active content types to prevent script execution
|
||||
# in the application origin when users open generated artifacts.
|
||||
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
|
||||
|
||||
if mime_type and mime_type.startswith("text/"):
|
||||
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
|
||||
|
||||
if is_text_file_by_content(actual_path):
|
||||
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
|
||||
|
||||
return Response(content=actual_path.read_bytes(), media_type=mime_type, headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)})
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/threads/{thread_id}/artifacts/{path:path}",
|
||||
summary="Update Artifact File",
|
||||
description="Update a text artifact file in the current thread workspace.",
|
||||
)
|
||||
@require_permission("threads", "write", owner_check=True)
|
||||
async def update_artifact(
|
||||
thread_id: str,
|
||||
path: str,
|
||||
request: Request,
|
||||
body: ArtifactUpdateRequest,
|
||||
) -> Response:
|
||||
"""Update a text artifact.
|
||||
|
||||
This endpoint is intentionally limited to normal text artifacts. Files inside
|
||||
.skill archives and active web content remain read-only from the artifact UI.
|
||||
"""
|
||||
if ".skill/" in path:
|
||||
raise HTTPException(status_code=400, detail="Skill archive contents are read-only.")
|
||||
|
||||
path = coerce_artifact_virtual_path(path)
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, path)
|
||||
logger.info(f"Updating artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
|
||||
|
||||
if not actual_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
|
||||
|
||||
if not actual_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(actual_path)
|
||||
suffix = actual_path.suffix.lower()
|
||||
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
raise HTTPException(status_code=400, detail="Active web artifacts are read-only.")
|
||||
|
||||
if suffix not in EDITABLE_TEXT_SUFFIXES and not (mime_type and mime_type.startswith("text/")):
|
||||
raise HTTPException(status_code=400, detail="Only text artifacts can be edited.")
|
||||
|
||||
actual_path.write_text(body.content, encoding="utf-8")
|
||||
return PlainTextResponse(content=body.content, media_type=mime_type or "text/plain")
|
||||
File diff suppressed because it is too large.
Load diff
@@ -55,6 +55,7 @@ from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
|
||||
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
@@ -254,7 +255,10 @@ def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None
|
||||
if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True):
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
|
||||
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
|
||||
# Event-driven refresh is incremental. Content hash + embedding
|
||||
# fingerprint decide which pages need encoding; unchanged Wiki pages
|
||||
# must retain and reuse their existing vectors.
|
||||
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=False))
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
@@ -496,15 +500,7 @@ def _should_follow_weknora_binary_redirect(path: str) -> bool:
|
||||
"""Keep protected preview/image bytes inside the DeerFlow proxy."""
|
||||
|
||||
segments = _path_segments(path)
|
||||
return (
|
||||
len(segments) == 5
|
||||
and segments[:3] == ["api", "v1", "knowledge"]
|
||||
and segments[4] == "preview"
|
||||
) or (
|
||||
len(segments) == 5
|
||||
and segments[:3] == ["api", "v1", "knowledge-bases"]
|
||||
and segments[4] == "files"
|
||||
)
|
||||
return (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge"] and segments[4] == "preview") or (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge-bases"] and segments[4] == "files")
|
||||
|
||||
|
||||
def _forward_request_headers(request: Request) -> dict[str, str]:
|
||||
@@ -1833,6 +1829,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"knowledge_count": int(source.get("knowledge_count") or 0),
|
||||
"chunk_count": int(source.get("chunk_count") or 0),
|
||||
"processing_count": int(source.get("processing_count") or 0),
|
||||
"share_count": int(source.get("share_count") or 0),
|
||||
"is_processing": bool(source.get("is_processing") or int(source.get("processing_count") or 0) > 0),
|
||||
"is_temporary": bool(source.get("is_temporary", False)),
|
||||
"embedding_model_id": source.get("embedding_model_id"),
|
||||
"vector_store_id": source.get("vector_store_id"),
|
||||
"vector_store_name": source.get("vector_store_name"),
|
||||
@@ -1840,16 +1839,23 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
|
||||
"vector_store_engine_type": source.get("vector_store_engine_type"),
|
||||
"vector_store_status": source.get("vector_store_status"),
|
||||
"chunking_config": source.get("chunking_config") if isinstance(source.get("chunking_config"), dict) else {},
|
||||
"image_processing_config": (source.get("image_processing_config") if isinstance(source.get("image_processing_config"), dict) else {}),
|
||||
"indexing_strategy": source.get("indexing_strategy") if isinstance(source.get("indexing_strategy"), dict) else {},
|
||||
"capabilities": source.get("capabilities") if isinstance(source.get("capabilities"), dict) else {},
|
||||
"extract_config": source.get("extract_config") if isinstance(source.get("extract_config"), dict) else {},
|
||||
"faq_config": source.get("faq_config") if isinstance(source.get("faq_config"), dict) else {},
|
||||
"wiki_config": source.get("wiki_config") if isinstance(source.get("wiki_config"), dict) else {},
|
||||
"question_generation_config": (source.get("question_generation_config") if isinstance(source.get("question_generation_config"), dict) else {}),
|
||||
"auto_tag_config": source.get("auto_tag_config") if isinstance(source.get("auto_tag_config"), dict) else {},
|
||||
"vlm_config": source.get("vlm_config") if isinstance(source.get("vlm_config"), dict) else {},
|
||||
"asr_config": source.get("asr_config") if isinstance(source.get("asr_config"), dict) else {},
|
||||
"summary_model_id": source.get("summary_model_id"),
|
||||
"storage_provider": (source.get("storage_provider_config", {}).get("provider") if isinstance(source.get("storage_provider_config"), dict) else None),
|
||||
"created_at": row.get("created_at"),
|
||||
"updated_at": row.get("updated_at"),
|
||||
"published_at": row.get("published_at"),
|
||||
"remote_created_at": source.get("created_at"),
|
||||
"remote_updated_at": source.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
@@ -1877,11 +1883,7 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
write=False,
|
||||
is_admin=False,
|
||||
)
|
||||
if (
|
||||
row is None
|
||||
or row.get("publication_status") != "published"
|
||||
or _is_conversation_deposit_mapping(row)
|
||||
):
|
||||
if row is None or row.get("publication_status") != "published" or _is_conversation_deposit_mapping(row):
|
||||
raise HTTPException(status_code=404, detail="Published knowledge base not found")
|
||||
return row
|
||||
|
||||
@@ -2042,6 +2044,7 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
|
||||
if not slug and not title and not content and not summary:
|
||||
return None
|
||||
return {
|
||||
"page_metadata": page.get("page_metadata") if isinstance(page.get("page_metadata"), dict) else {},
|
||||
"id": str(page.get("id") or ""),
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
@@ -2537,24 +2540,260 @@ async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest)
|
||||
return {"success": True, "provider": "weknora", "health": health}
|
||||
|
||||
|
||||
_PUBLIC_WIKI_SUMMARY_CONCURRENCY = 8
|
||||
_PUBLIC_WIKI_PAGE_SIZE = 500
|
||||
_PUBLIC_WIKI_PAGE_LIMIT = 100
|
||||
_GENERATED_PUBLIC_WIKI_INDEX_SLUG = "__index__"
|
||||
|
||||
|
||||
def _is_public_wiki_index_page(page: dict[str, Any]) -> bool:
|
||||
page_type = str(page.get("page_type") or "").strip().lower()
|
||||
title = str(page.get("title") or "").strip().lower()
|
||||
slug_name = str(page.get("slug") or "").strip("/").rsplit("/", 1)[-1].lower()
|
||||
return page_type == "index" or title in {"索引", "index"} or slug_name in {"index", "home", "readme"}
|
||||
|
||||
|
||||
def _public_wiki_default_page(pages: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
return (
|
||||
next((page for page in pages if _is_public_wiki_index_page(page)), None)
|
||||
or next((page for page in pages if str(page.get("page_type") or "").lower() == "overview"), None)
|
||||
or next(
|
||||
(page for page in pages if not str(page.get("parent_slug") or "").strip() and int(page.get("depth") or 0) == 0),
|
||||
None,
|
||||
)
|
||||
or (pages[0] if pages else None)
|
||||
)
|
||||
|
||||
|
||||
def _count_public_wiki_values(pages: list[dict[str, Any]], key: str, fallback: str) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for page in pages:
|
||||
value = str(page.get(key) or fallback).strip().lower() or fallback
|
||||
counts[value] = counts.get(value, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def _public_wiki_time_bound(pages: list[dict[str, Any]], key: str, *, latest: bool) -> str | None:
|
||||
values = [str(page.get(key) or "").strip() for page in pages]
|
||||
values = [value for value in values if value]
|
||||
if not values:
|
||||
return None
|
||||
return max(values) if latest else min(values)
|
||||
|
||||
|
||||
async def _all_public_wiki_page_views(
|
||||
client: WeKnoraClient,
|
||||
remote_id: str,
|
||||
) -> tuple[list[dict[str, Any]], int, bool]:
|
||||
pages_by_slug: dict[str, dict[str, Any]] = {}
|
||||
remote_total = 0
|
||||
complete = False
|
||||
for page_number in range(1, _PUBLIC_WIKI_PAGE_LIMIT + 1):
|
||||
result = await client.list_wiki_pages(
|
||||
remote_id,
|
||||
page=page_number,
|
||||
page_size=_PUBLIC_WIKI_PAGE_SIZE,
|
||||
query="",
|
||||
)
|
||||
raw_pages = result.get("pages") if isinstance(result, dict) else []
|
||||
page_items = raw_pages if isinstance(raw_pages, list) else []
|
||||
for page_item in page_items:
|
||||
if not isinstance(page_item, dict):
|
||||
continue
|
||||
page = _wiki_page_view(page_item)
|
||||
if page is None:
|
||||
continue
|
||||
slug = str(page.get("slug") or "").strip()
|
||||
pages_by_slug[slug or f"__page_{len(pages_by_slug)}"] = page
|
||||
reported_total = int(result.get("total") or 0)
|
||||
remote_total = max(remote_total, reported_total, len(pages_by_slug))
|
||||
total_pages = max(1, int(result.get("total_pages") or 1))
|
||||
if not page_items or page_number >= total_pages or (reported_total > 0 and len(pages_by_slug) >= remote_total):
|
||||
complete = True
|
||||
break
|
||||
return list(pages_by_slug.values()), max(remote_total, len(pages_by_slug)), complete
|
||||
|
||||
|
||||
def _public_wiki_summary(
|
||||
mapping_id: str,
|
||||
pages: list[dict[str, Any]],
|
||||
*,
|
||||
total: int,
|
||||
complete: bool,
|
||||
) -> dict[str, Any]:
|
||||
explicit_index = next((page for page in pages if _is_public_wiki_index_page(page)), None)
|
||||
default_page = _public_wiki_default_page(pages)
|
||||
if explicit_index is not None:
|
||||
index_mode = "explicit"
|
||||
index_slug: str | None = str(explicit_index.get("slug") or "") or None
|
||||
elif pages:
|
||||
index_mode = "generated"
|
||||
index_slug = _GENERATED_PUBLIC_WIKI_INDEX_SLUG
|
||||
else:
|
||||
index_mode = "none"
|
||||
index_slug = None
|
||||
default_slug = index_slug or (str(default_page.get("slug") or "") if default_page else None)
|
||||
route = f"/embed/knowledge/{quote(mapping_id, safe='')}"
|
||||
if default_slug:
|
||||
route = f"{route}?{urlencode({'wikiSlug': default_slug})}"
|
||||
return {
|
||||
"status": "ready" if pages else "empty",
|
||||
"page_count": total,
|
||||
"loaded_page_count": len(pages),
|
||||
"stats_complete": complete,
|
||||
"page_type_counts": _count_public_wiki_values(pages, "page_type", "page"),
|
||||
"page_status_counts": _count_public_wiki_values(pages, "status", "unknown"),
|
||||
"index_mode": index_mode,
|
||||
"has_explicit_index": explicit_index is not None,
|
||||
"index_slug": index_slug,
|
||||
"default_slug": default_slug,
|
||||
"created_at": _public_wiki_time_bound(pages, "created_at", latest=False),
|
||||
"updated_at": _public_wiki_time_bound(pages, "updated_at", latest=True),
|
||||
"embed_route": route,
|
||||
}
|
||||
|
||||
|
||||
def _unavailable_public_wiki_summary(mapping_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"page_count": 0,
|
||||
"loaded_page_count": 0,
|
||||
"stats_complete": False,
|
||||
"page_type_counts": {},
|
||||
"page_status_counts": {},
|
||||
"index_mode": "unknown",
|
||||
"has_explicit_index": False,
|
||||
"index_slug": None,
|
||||
"default_slug": None,
|
||||
"created_at": None,
|
||||
"updated_at": None,
|
||||
"embed_route": f"/embed/knowledge/{quote(mapping_id, safe='')}",
|
||||
}
|
||||
|
||||
|
||||
def _public_local_index_summary(
|
||||
request: Request,
|
||||
mapping: dict[str, Any],
|
||||
state: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
app_state = request.app.state
|
||||
local_config = getattr(
|
||||
getattr(getattr(app_state, "config", None), "llmwiki", None),
|
||||
"local_wiki_index",
|
||||
None,
|
||||
)
|
||||
enabled = bool(getattr(local_config, "enabled", False))
|
||||
current = dict(state or {})
|
||||
state_name = str(current.get("state") or "not_synced")
|
||||
if state_name == "idle" and int(current.get("index_revision") or 0) == 0 and not current.get("last_completed_at") and not current.get("last_success_at"):
|
||||
state_name = "not_synced"
|
||||
embedding = getattr(app_state, "llmwiki_embedding", None)
|
||||
fingerprint = embedding.fingerprint if embedding is not None else None
|
||||
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(
|
||||
current,
|
||||
fingerprint,
|
||||
)
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"wiki_index_enabled": bool(mapping.get("wiki_index_enabled", True)),
|
||||
"state": state_name,
|
||||
"fully_vectorized": fully_vectorized,
|
||||
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
|
||||
"remote_page_count": int(current.get("remote_page_count") or 0),
|
||||
"local_page_count": int(current.get("local_page_count") or 0),
|
||||
"ready_page_count": int(current.get("ready_page_count") or 0),
|
||||
"failed_page_count": int(current.get("failed_page_count") or 0),
|
||||
"vector_count": int(current.get("vector_count") or 0),
|
||||
"last_started_at": current.get("last_started_at"),
|
||||
"last_completed_at": current.get("last_completed_at"),
|
||||
"last_success_at": current.get("last_success_at"),
|
||||
"updated_at": current.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/public/llmwiki/knowledge-bases")
|
||||
async def list_public_knowledge_bases(request: Request) -> dict[str, Any]:
|
||||
"""List the minimal identity of every knowledge base published as a public Wiki."""
|
||||
"""List live, published Wiki libraries with safe metadata and index health."""
|
||||
|
||||
rows = await _store(request).list_visible(
|
||||
"__public_wiki__",
|
||||
scope="public",
|
||||
is_admin=False,
|
||||
)
|
||||
knowledge_bases = [
|
||||
{
|
||||
"id": str(row["id"]),
|
||||
"name": str(row.get("name") or ""),
|
||||
}
|
||||
for row in rows
|
||||
if row.get("publication_status") == "published"
|
||||
and not _is_conversation_deposit_mapping(row)
|
||||
rows = [
|
||||
row
|
||||
for row in await _store(request).list_visible(
|
||||
"__public_wiki__",
|
||||
scope="public",
|
||||
is_admin=False,
|
||||
)
|
||||
if row.get("publication_status") == "published" and not _is_conversation_deposit_mapping(row)
|
||||
]
|
||||
client = _client_or_503()
|
||||
try:
|
||||
remote_items = await client.list_knowledge_bases()
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
remote_by_id = {str(item.get("id")): item for item in remote_items if isinstance(item, dict) and item.get("id")}
|
||||
# A successful upstream list is authoritative: stale DeerFlow mappings are
|
||||
# omitted so discovery never advertises a Wiki that its detail route cannot open.
|
||||
rows = [row for row in rows if str(row.get("weknora_id") or "") in remote_by_id]
|
||||
mapping_ids = [str(row["id"]) for row in rows]
|
||||
index_states: dict[str, dict[str, Any]] = {}
|
||||
index_store = getattr(request.app.state, "llmwiki_index_store", None)
|
||||
if index_store is not None and mapping_ids:
|
||||
try:
|
||||
index_states = {str(state.get("knowledge_base_mapping_id") or ""): state for state in await index_store.list_index_status(mapping_ids) if isinstance(state, dict)}
|
||||
except Exception:
|
||||
logger.warning("Failed to read public Wiki local-index summaries", exc_info=True)
|
||||
|
||||
semaphore = asyncio.Semaphore(_PUBLIC_WIKI_SUMMARY_CONCURRENCY)
|
||||
|
||||
async def build_summary(row: dict[str, Any]) -> dict[str, Any]:
|
||||
mapping_id = str(row["id"])
|
||||
remote_id = str(row["weknora_id"])
|
||||
remote = remote_by_id[remote_id]
|
||||
async with semaphore:
|
||||
try:
|
||||
remote = {**remote, **await client.get_knowledge_base(remote_id)}
|
||||
except WeKnoraError as exc:
|
||||
if exc.status_code == 404:
|
||||
return {}
|
||||
logger.warning(
|
||||
"Failed to load public Wiki detail metadata: mapping_id=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
pages, page_total, complete = await _all_public_wiki_page_views(client, remote_id)
|
||||
wiki = _public_wiki_summary(
|
||||
mapping_id,
|
||||
pages,
|
||||
total=page_total,
|
||||
complete=complete,
|
||||
)
|
||||
except WeKnoraError as exc:
|
||||
logger.warning(
|
||||
"Failed to load public Wiki page summary: mapping_id=%s",
|
||||
mapping_id,
|
||||
exc_info=True,
|
||||
)
|
||||
wiki = (
|
||||
_public_wiki_summary(mapping_id, [], total=0, complete=True)
|
||||
if exc.status_code == 404
|
||||
else _unavailable_public_wiki_summary(mapping_id)
|
||||
)
|
||||
return {
|
||||
**_mapping_view(
|
||||
row,
|
||||
remote,
|
||||
actor_user_id="__public_wiki__",
|
||||
is_admin=False,
|
||||
),
|
||||
"wiki": wiki,
|
||||
"local_index": _public_local_index_summary(
|
||||
request,
|
||||
row,
|
||||
index_states.get(mapping_id),
|
||||
),
|
||||
}
|
||||
|
||||
knowledge_bases = [summary for summary in await asyncio.gather(*(build_summary(row) for row in rows)) if summary]
|
||||
return {"knowledge_bases": knowledge_bases, "total": len(knowledge_bases)}
|
||||
|
||||
|
||||
@@ -2596,13 +2835,7 @@ async def list_public_wiki_pages(
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
raw_pages = result.get("pages") if isinstance(result, dict) else []
|
||||
pages = [
|
||||
view
|
||||
for page_item in (raw_pages if isinstance(raw_pages, list) else [])
|
||||
if isinstance(page_item, dict)
|
||||
for view in [_wiki_page_view(page_item)]
|
||||
if view is not None
|
||||
]
|
||||
pages = [view for page_item in (raw_pages if isinstance(raw_pages, list) else []) if isinstance(page_item, dict) for view in [_wiki_page_view(page_item)] if view is not None]
|
||||
return {**result, "pages": pages}
|
||||
|
||||
|
||||
@@ -2650,11 +2883,7 @@ def _public_wiki_index_item(value: Any) -> dict[str, Any] | None:
|
||||
"slug": slug,
|
||||
"title": str(value.get("title") or slug),
|
||||
"summary": str(value.get("summary") or ""),
|
||||
"category_path": (
|
||||
[str(item) for item in value.get("category_path", []) if str(item).strip()]
|
||||
if isinstance(value.get("category_path"), list)
|
||||
else []
|
||||
),
|
||||
"category_path": ([str(item) for item in value.get("category_path", []) if str(item).strip()] if isinstance(value.get("category_path"), list) else []),
|
||||
"wiki_path": str(value.get("wiki_path") or ""),
|
||||
"depth": int(value.get("depth") or 0),
|
||||
}
|
||||
@@ -2680,20 +2909,12 @@ async def _all_public_wiki_index(client: WeKnoraClient, remote_id: str) -> dict[
|
||||
version = max(version, int(payload.get("version") or 0))
|
||||
raw_groups = payload.get("groups") if isinstance(payload.get("groups"), list) else []
|
||||
group = next(
|
||||
(
|
||||
value
|
||||
for value in raw_groups
|
||||
if isinstance(value, dict) and value.get("type") == page_type
|
||||
),
|
||||
(value for value in raw_groups if isinstance(value, dict) and value.get("type") == page_type),
|
||||
{},
|
||||
)
|
||||
total = max(total, int(group.get("total") or 0))
|
||||
raw_items = group.get("items") if isinstance(group.get("items"), list) else []
|
||||
items.extend(
|
||||
item
|
||||
for value in raw_items
|
||||
if (item := _public_wiki_index_item(value)) is not None
|
||||
)
|
||||
items.extend(item for value in raw_items if (item := _public_wiki_index_item(value)) is not None)
|
||||
next_cursor = str(group.get("next_cursor") or "")
|
||||
if not next_cursor or next_cursor in seen_cursors:
|
||||
break
|
||||
@@ -2770,12 +2991,11 @@ async def list_knowledge_bases(
|
||||
knowledge_bases: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
remote_item = remote.get(str(row["weknora_id"]))
|
||||
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
|
||||
# Owners need to see stale personal mappings so they can clean them up.
|
||||
# Public/selector views must never expose a resource that no longer exists
|
||||
# in WeKnora.
|
||||
if remote_item is None and scope != "personal":
|
||||
# A successful remote list is authoritative for every scope. Keep the
|
||||
# mapping for existing references, but never display a deleted base.
|
||||
if remote_item is None:
|
||||
continue
|
||||
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
|
||||
knowledge_bases.append(_mapping_view(row, remote_item, actor_user_id=user_id, is_admin=is_admin))
|
||||
return {"knowledge_bases": knowledge_bases}
|
||||
|
||||
@@ -2841,6 +3061,7 @@ async def create_knowledge_base(request: Request, body: KnowledgeBaseCreate) ->
|
||||
except Exception:
|
||||
logger.exception("Failed to compensate orphan WeKnora knowledge base")
|
||||
raise HTTPException(status_code=500, detail="Failed to register the knowledge base") from None
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return _mapping_view(row, remote, actor_user_id=user_id, is_admin=is_admin)
|
||||
|
||||
|
||||
@@ -2860,13 +3081,19 @@ async def get_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any
|
||||
async def update_knowledge_base(request: Request, mapping_id: str, body: KnowledgeBaseUpdate) -> dict[str, Any]:
|
||||
row, user_id, is_admin = await _authorized_mapping(request, mapping_id, write=True)
|
||||
changes = body.model_dump(exclude_none=True)
|
||||
if "name" in changes:
|
||||
changes["name"] = changes["name"].strip()
|
||||
if not changes["name"]:
|
||||
raise HTTPException(status_code=422, detail="知识库名称不能为空")
|
||||
if changes["name"] == CONVERSATION_DEPOSIT_KB_NAME:
|
||||
raise HTTPException(status_code=409, detail="对话沉淀是 cmzs 系统保留知识库")
|
||||
if not changes:
|
||||
return _mapping_view(row, None, actor_user_id=user_id, is_admin=is_admin)
|
||||
try:
|
||||
remote = await _client_or_503().update_knowledge_base(str(row["weknora_id"]), changes)
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
updated = await _store(request).update_mapping(mapping_id, name=body.name, description=body.description)
|
||||
updated = await _store(request).update_mapping(mapping_id, name=changes.get("name"), description=changes.get("description"))
|
||||
return _mapping_view(updated or row, remote, actor_user_id=user_id, is_admin=is_admin)
|
||||
|
||||
|
||||
@@ -3122,12 +3349,14 @@ async def upload_document(request: Request, mapping_id: str, file: UploadFile =
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=422, detail="File name is required")
|
||||
try:
|
||||
return await _client_or_503().upload_document(
|
||||
result = await _client_or_503().upload_document(
|
||||
str(row["weknora_id"]),
|
||||
filename=file.filename,
|
||||
content=content,
|
||||
content_type=file.content_type or "application/octet-stream",
|
||||
)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return result
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
|
||||
@@ -3139,7 +3368,9 @@ async def import_document_url(request: Request, mapping_id: str, body: Knowledge
|
||||
if not url.lower().startswith(("http://", "https://")):
|
||||
raise HTTPException(status_code=422, detail="URL must start with http:// or https://")
|
||||
try:
|
||||
return await _client_or_503().import_document_url(str(row["weknora_id"]), url=url)
|
||||
result = await _client_or_503().import_document_url(str(row["weknora_id"]), url=url)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return result
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
|
||||
@@ -3148,11 +3379,13 @@ async def import_document_url(request: Request, mapping_id: str, body: Knowledge
|
||||
async def create_manual_document(request: Request, mapping_id: str, body: ManualKnowledgeCreate) -> dict[str, Any]:
|
||||
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
|
||||
try:
|
||||
return await _client_or_503().create_manual_document(
|
||||
result = await _client_or_503().create_manual_document(
|
||||
str(row["weknora_id"]),
|
||||
title=body.title.strip(),
|
||||
content=body.content,
|
||||
)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return result
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
|
||||
@@ -3164,6 +3397,7 @@ async def delete_document(request: Request, mapping_id: str, document_id: str) -
|
||||
try:
|
||||
await _verify_document(client, row, document_id)
|
||||
await client.delete_document(document_id)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
|
||||
@@ -3174,7 +3408,9 @@ async def reprocess_document(request: Request, mapping_id: str, document_id: str
|
||||
client = _client_or_503()
|
||||
try:
|
||||
await _verify_document(client, row, document_id)
|
||||
return await client.reprocess_document(document_id)
|
||||
result = await client.reprocess_document(document_id)
|
||||
_schedule_local_wiki_sync(request, row)
|
||||
return result
|
||||
except WeKnoraError as exc:
|
||||
_raise_upstream(exc)
|
||||
|
||||
@@ -3201,12 +3437,7 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
|
||||
except WikiIndexError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
result["results"] = [
|
||||
_sanitize_conversation_deposit_value(item)
|
||||
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
|
||||
else item
|
||||
for item in result.get("results") or []
|
||||
]
|
||||
result["results"] = [_sanitize_conversation_deposit_value(item) if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {})) else item for item in result.get("results") or []]
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -2,15 +2,20 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi import APIRouter, File, Form, HTTPException, Query, Request, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
from deerflow.config.runtime_paths import resolve_path, runtime_home
|
||||
from deerflow.config.system_settings import load_system_settings, save_system_settings
|
||||
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
|
||||
@@ -37,6 +42,10 @@ class WikiIndexSettingsRequest(BaseModel):
|
||||
external_search_enabled: bool | None = None
|
||||
|
||||
|
||||
class LocalEncoderConfigRequest(BaseModel):
|
||||
model_path: str = Field(min_length=1, max_length=4096)
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
@@ -55,6 +64,149 @@ def _services(request: Request):
|
||||
return config, search, index_store
|
||||
|
||||
|
||||
def _local_encoder(request: Request):
|
||||
config = request.app.state.config.llmwiki.local_wiki_index.embedding
|
||||
embedding = getattr(request.app.state, "llmwiki_embedding", None)
|
||||
if config.provider != "local":
|
||||
raise HTTPException(status_code=409, detail={"code": "EMBEDDING_NOT_LOCAL", "message": "当前 embedding provider 不是 local"})
|
||||
if embedding is None:
|
||||
raise HTTPException(status_code=503, detail={"code": "LOCAL_ENCODER_UNAVAILABLE", "message": "本地编码器未配置或未启用"})
|
||||
return embedding
|
||||
|
||||
|
||||
def _validated_encoder_path(raw_path: str) -> Path:
|
||||
path = resolve_path(raw_path.strip())
|
||||
if not path.is_dir():
|
||||
raise HTTPException(status_code=422, detail="编码器目录不存在")
|
||||
if not ((path / "onnx" / "model.onnx").is_file() or (path / "model.onnx").is_file()):
|
||||
raise HTTPException(status_code=422, detail="编码器目录缺少 onnx/model.onnx 或 model.onnx")
|
||||
return path
|
||||
|
||||
|
||||
def _safe_relative_upload_path(value: str) -> Path:
|
||||
normalized = str(value or "").replace("\\", "/").strip("/")
|
||||
candidate = PurePosixPath(normalized)
|
||||
if not normalized or candidate.is_absolute() or ".." in candidate.parts:
|
||||
raise HTTPException(status_code=422, detail=f"非法模型文件路径:{value}")
|
||||
if len(normalized) > 512:
|
||||
raise HTTPException(status_code=422, detail="模型文件路径过长")
|
||||
return Path(*candidate.parts)
|
||||
|
||||
|
||||
def _copy_encoder_uploads(files: list[UploadFile], paths: list[str], staging: Path) -> int:
|
||||
total = 0
|
||||
for upload, raw_path in zip(files, paths, strict=True):
|
||||
relative = _safe_relative_upload_path(raw_path)
|
||||
destination = staging / relative
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
upload.file.seek(0)
|
||||
with destination.open("wb") as output:
|
||||
while chunk := upload.file.read(4 * 1024 * 1024):
|
||||
output.write(chunk)
|
||||
total += len(chunk)
|
||||
return total
|
||||
|
||||
|
||||
def _activate_uploaded_encoder(staging: Path) -> Path:
|
||||
model_root = (runtime_home() / "models").resolve()
|
||||
model_root.mkdir(parents=True, exist_ok=True)
|
||||
active = (model_root / "bge-m3").resolve()
|
||||
if active.parent != model_root:
|
||||
raise RuntimeError("编码器目标目录越界")
|
||||
backup = (model_root / f".bge-m3-backup-{uuid4().hex}").resolve()
|
||||
if active.exists():
|
||||
active.replace(backup)
|
||||
try:
|
||||
staging.replace(active)
|
||||
except Exception:
|
||||
if backup.exists() and not active.exists():
|
||||
backup.replace(active)
|
||||
raise
|
||||
if backup.exists():
|
||||
shutil.rmtree(backup)
|
||||
return active
|
||||
|
||||
|
||||
async def _save_encoder_path(request: Request, path: Path) -> dict[str, object]:
|
||||
embedding = _local_encoder(request)
|
||||
current = load_system_settings()
|
||||
encoder_settings = current.local_wiki_encoder.model_copy(update={"model_path": str(path)})
|
||||
save_system_settings(current.model_copy(update={"local_wiki_encoder": encoder_settings}))
|
||||
await embedding.reset_local_encoder(model_path=str(path))
|
||||
return embedding.local_encoder_status()
|
||||
|
||||
|
||||
@router.get("/local-encoder/status")
|
||||
async def local_encoder_status(request: Request) -> dict[str, object]:
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
return _local_encoder(request).local_encoder_status()
|
||||
|
||||
|
||||
@router.post("/local-encoder/start", status_code=202)
|
||||
async def start_local_encoder(request: Request) -> dict[str, object]:
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
try:
|
||||
return _local_encoder(request).start_local_encoder()
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail={"code": "LOCAL_ENCODER_START_REJECTED", "message": str(exc)}) from None
|
||||
|
||||
|
||||
@router.put("/local-encoder/config")
|
||||
async def configure_local_encoder(request: Request, body: LocalEncoderConfigRequest) -> dict[str, object]:
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
return await _save_encoder_path(request, _validated_encoder_path(body.model_path))
|
||||
|
||||
|
||||
@router.post("/local-encoder/upload")
|
||||
async def upload_local_encoder(
|
||||
request: Request,
|
||||
files: list[UploadFile] = File(...),
|
||||
paths: list[str] = Form(...),
|
||||
) -> dict[str, object]:
|
||||
"""Stream an offline model directory into runtime storage and activate it.
|
||||
|
||||
Starlette spools multipart files to disk, and the copy loop is chunked, so
|
||||
multi-gigabyte ONNX models do not need to be loaded into Gateway memory.
|
||||
"""
|
||||
|
||||
_, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
if not files or len(files) != len(paths):
|
||||
raise HTTPException(status_code=422, detail="模型文件与相对路径不匹配")
|
||||
if len(files) > 500:
|
||||
raise HTTPException(status_code=422, detail="模型目录文件数不能超过 500")
|
||||
|
||||
staging_root = (runtime_home() / "model-uploads").resolve()
|
||||
staging = (staging_root / uuid4().hex).resolve()
|
||||
if staging.parent != staging_root:
|
||||
raise HTTPException(status_code=500, detail="模型暂存目录越界")
|
||||
staging.mkdir(parents=True, exist_ok=False)
|
||||
try:
|
||||
uploaded_bytes = await asyncio.to_thread(_copy_encoder_uploads, files, paths, staging)
|
||||
_validated_encoder_path(str(staging))
|
||||
await _local_encoder(request).reset_local_encoder()
|
||||
active = await asyncio.to_thread(_activate_uploaded_encoder, staging)
|
||||
status = await _save_encoder_path(request, active)
|
||||
return {**status, "uploaded_files": len(files), "uploaded_bytes": uploaded_bytes}
|
||||
except HTTPException:
|
||||
raise
|
||||
except OSError as exc:
|
||||
logger.exception("Failed to store uploaded local encoder")
|
||||
raise HTTPException(status_code=507, detail=f"编码器文件保存失败:{exc}") from None
|
||||
finally:
|
||||
for upload in files:
|
||||
await upload.close()
|
||||
if staging.exists():
|
||||
await asyncio.to_thread(shutil.rmtree, staging, True)
|
||||
|
||||
|
||||
async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tuple[list[dict[str, Any]], str, bool]:
|
||||
user_id, is_admin = await _actor(request)
|
||||
mapping_store = request.app.state.llmwiki_store
|
||||
@@ -313,7 +465,7 @@ async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[st
|
||||
if not mapping.get("wiki_index_enabled", True):
|
||||
mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
|
||||
job_id = str(uuid4())
|
||||
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
|
||||
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=False))
|
||||
return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
"""User-facing skill distillation APIs plus administrator maintenance APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Literal
|
||||
|
||||
from croniter import croniter
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from app.gateway.skill_knowledge_job_executor import _skill_directory
|
||||
from deerflow.config.system_settings import (
|
||||
SkillKnowledgeAutoSyncSettings,
|
||||
load_system_settings,
|
||||
save_system_settings,
|
||||
)
|
||||
from deerflow.integrations.weknora.runtime import (
|
||||
build_weknora_client,
|
||||
get_resolved_llmwiki_runtime,
|
||||
)
|
||||
from deerflow.skill_knowledge import scan_skill
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.types import SkillCategory
|
||||
|
||||
router = APIRouter(prefix="/api/skill-knowledge", tags=["skill-knowledge"])
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return "system", True
|
||||
return str(user.id), getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> str:
|
||||
actor, is_admin = await _actor(request)
|
||||
if not is_admin:
|
||||
raise HTTPException(status_code=403, detail="技能知识归纳仅管理员可操作")
|
||||
return actor
|
||||
|
||||
|
||||
def _owned_by_actor(row: dict[str, Any] | None, actor: str, is_admin: bool) -> bool:
|
||||
return bool(row and (is_admin or str(row.get("created_by") or "") == actor))
|
||||
|
||||
|
||||
def _store(request: Request):
|
||||
value = getattr(request.app.state, "skill_knowledge_store", None)
|
||||
if value is None:
|
||||
raise HTTPException(status_code=503, detail="技能知识归纳存储不可用")
|
||||
return value
|
||||
|
||||
|
||||
class SyncTarget(BaseModel):
|
||||
target_type: Literal["weknora", "assistant"]
|
||||
target_mode: Literal["wiki", "document", "faq", "conversation_archive", "readonly_sidecar", "assistant"]
|
||||
target_id: str = Field(min_length=1, max_length=128)
|
||||
target_name: str = Field(default="", max_length=255)
|
||||
remote_business_key: str | None = Field(default=None, max_length=512)
|
||||
readonly: bool = False
|
||||
|
||||
|
||||
class CreateSyncJobRequest(BaseModel):
|
||||
skill_names: list[str] = Field(min_length=1, max_length=200)
|
||||
targets: list[SyncTarget] = Field(min_length=1, max_length=1)
|
||||
force: bool = False
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] = "off"
|
||||
|
||||
@field_validator("skill_names")
|
||||
@classmethod
|
||||
def validate_names(cls, values: list[str]) -> list[str]:
|
||||
result = list(dict.fromkeys(value.strip() for value in values if value.strip()))
|
||||
if not result:
|
||||
raise ValueError("At least one skill is required")
|
||||
return result
|
||||
|
||||
|
||||
class ReviewRequest(BaseModel):
|
||||
review_ids: list[str] = Field(min_length=1, max_length=500)
|
||||
action: Literal["approve", "reject"]
|
||||
edits: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
||||
reason: str | None = Field(default=None, max_length=1024)
|
||||
|
||||
|
||||
class DetachRequest(BaseModel):
|
||||
delete_contributions: bool
|
||||
|
||||
|
||||
class AutoSyncSettingsUpdate(BaseModel):
|
||||
watch_enabled: bool | None = None
|
||||
schedule_enabled: bool | None = None
|
||||
schedule_cron: str | None = Field(default=None, min_length=5, max_length=128)
|
||||
debounce_seconds: int | None = Field(default=None, ge=10, le=86_400)
|
||||
write_remote_enabled: bool | None = None
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] | None = None
|
||||
|
||||
@field_validator("schedule_cron")
|
||||
@classmethod
|
||||
def validate_cron(cls, value: str | None) -> str | None:
|
||||
if value is not None and not croniter.is_valid(value):
|
||||
raise ValueError("无效的 cron 表达式")
|
||||
return value
|
||||
|
||||
|
||||
@router.get("/bindings")
|
||||
async def list_bindings(
|
||||
request: Request,
|
||||
skill_names: list[str] | None = Query(default=None),
|
||||
target_id: str | None = Query(default=None),
|
||||
include_detached: bool = Query(default=False),
|
||||
mine_only: bool = Query(default=False),
|
||||
) -> dict[str, Any]:
|
||||
actor, is_admin = await _actor(request)
|
||||
rows = await _store(request).list_bindings(
|
||||
skill_names=skill_names,
|
||||
target_id=target_id,
|
||||
include_detached=include_detached,
|
||||
created_by=actor if mine_only or not is_admin else None,
|
||||
)
|
||||
return {"bindings": rows}
|
||||
|
||||
|
||||
@router.get("/source-status")
|
||||
async def source_status(request: Request, skill_names: list[str] = Query(...)) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
bindings = await _store(request).list_bindings(skill_names=skill_names)
|
||||
by_skill: dict[str, dict[str, Any]] = {}
|
||||
for name in list(dict.fromkeys(skill_names)):
|
||||
try:
|
||||
root = _skill_directory(request.app.state.config, name)
|
||||
result = await asyncio.to_thread(scan_skill, name, root)
|
||||
by_skill[name] = {"source_digest": result.source_digest, "counts": result.counts, "blocked_files": result.blocked_files}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
by_skill[name] = {"error": str(exc)}
|
||||
for binding in bindings:
|
||||
status = by_skill.get(binding["skill_name"])
|
||||
if status and status.get("source_digest") and status["source_digest"] != binding.get("synced_digest"):
|
||||
binding["computed_status"] = "stale"
|
||||
return {"skills": by_skill, "bindings": bindings}
|
||||
|
||||
|
||||
@router.get("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
|
||||
async def get_auto_sync_settings(request: Request) -> SkillKnowledgeAutoSyncSettings:
|
||||
await _require_admin(request)
|
||||
return load_system_settings().skill_knowledge_auto_sync
|
||||
|
||||
|
||||
@router.put("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
|
||||
async def update_auto_sync_settings(request: Request, body: AutoSyncSettingsUpdate) -> SkillKnowledgeAutoSyncSettings:
|
||||
await _require_admin(request)
|
||||
current = load_system_settings()
|
||||
updated = current.skill_knowledge_auto_sync.model_copy(update=body.model_dump(exclude_unset=True))
|
||||
saved = save_system_settings(current.model_copy(update={"skill_knowledge_auto_sync": updated}))
|
||||
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
|
||||
if service is not None:
|
||||
service.nudge()
|
||||
return saved.skill_knowledge_auto_sync
|
||||
|
||||
|
||||
async def _set_binding_auto_sync(request: Request, binding_id: str, *, enabled: bool) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
row = await _store(request).set_binding_auto_sync(binding_id, enabled=enabled)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
|
||||
if service is not None:
|
||||
service.nudge()
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/enable-auto-sync")
|
||||
async def enable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
return await _set_binding_auto_sync(request, binding_id, enabled=True)
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/disable-auto-sync")
|
||||
async def disable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
return await _set_binding_auto_sync(request, binding_id, enabled=False)
|
||||
|
||||
|
||||
@router.post("/sync-jobs", status_code=202)
|
||||
async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[str, Any]:
|
||||
actor, is_admin = await _actor(request)
|
||||
# Built-in and legacy skills need not have an ownership row. Validate
|
||||
# against the same running filesystem used by the skills list/executor.
|
||||
missing = []
|
||||
favorite_names: set[str] = set()
|
||||
custom_root: Any = None
|
||||
if not is_admin:
|
||||
favorite_names = set(await request.app.state.skill_store.list_favorite_skill_names(actor))
|
||||
storage_root = get_or_new_skill_storage(
|
||||
app_config=request.app.state.config
|
||||
).get_skills_root_path().resolve()
|
||||
custom_root = (storage_root / SkillCategory.CUSTOM.value).resolve()
|
||||
for name in body.skill_names:
|
||||
try:
|
||||
skill_path = await asyncio.to_thread(_skill_directory, request.app.state.config, name)
|
||||
if not is_admin:
|
||||
visible = await request.app.state.skill_store.get_visible(name, actor)
|
||||
is_public_filesystem_skill = not skill_path.resolve().is_relative_to(custom_root)
|
||||
if visible is None and name not in favorite_names and not is_public_filesystem_skill:
|
||||
missing.append(name)
|
||||
except (FileNotFoundError, ValueError):
|
||||
missing.append(name)
|
||||
if missing:
|
||||
raise HTTPException(status_code=404, detail=f"技能不存在:{', '.join(missing[:10])}")
|
||||
targets: list[dict[str, Any]] = []
|
||||
for target in body.targets:
|
||||
data = target.model_dump()
|
||||
if target.target_type != "weknora":
|
||||
raise HTTPException(status_code=422, detail="不支持技能直接归纳到助手知识库,请先归纳到 WeKnora 普通知识库")
|
||||
if target.target_mode != "wiki":
|
||||
raise HTTPException(status_code=422, detail="技能归纳目标仅支持 WeKnora Wiki 知识库")
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
target.target_id,
|
||||
actor,
|
||||
write=True,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail=f"普通知识库不存在或当前用户不可写:{target.target_id}")
|
||||
data["target_name"] = target.target_name or str(mapping.get("name") or target.target_id)
|
||||
data["remote_business_key"] = target.remote_business_key or f"weknora:kb:{mapping['weknora_id']}"
|
||||
targets.append(data)
|
||||
job = await _store(request).create_job(
|
||||
skill_names=body.skill_names,
|
||||
targets=targets,
|
||||
created_by=actor,
|
||||
force=body.force,
|
||||
review_mode=body.review_mode if is_admin else "off",
|
||||
)
|
||||
dispatcher = getattr(request.app.state, "skill_knowledge_dispatcher", None)
|
||||
if dispatcher is not None:
|
||||
dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.get("/sync-jobs")
|
||||
async def list_sync_jobs(
|
||||
request: Request,
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
mine_only: bool = Query(default=False),
|
||||
) -> dict[str, Any]:
|
||||
actor, is_admin = await _actor(request)
|
||||
return {
|
||||
"jobs": await _store(request).list_jobs(
|
||||
limit=limit,
|
||||
created_by=actor if mine_only or not is_admin else None,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@router.get("/sync-jobs/{job_id}")
|
||||
async def get_sync_job(request: Request, job_id: str) -> dict[str, Any]:
|
||||
actor, is_admin = await _actor(request)
|
||||
row = await _store(request).get_job(job_id)
|
||||
if not _owned_by_actor(row, actor, is_admin):
|
||||
raise HTTPException(status_code=404, detail="归纳任务不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/sync-jobs/{job_id}/cancel")
|
||||
async def cancel_sync_job(request: Request, job_id: str) -> dict[str, bool]:
|
||||
actor, is_admin = await _actor(request)
|
||||
row = await _store(request).get_job(job_id)
|
||||
if not _owned_by_actor(row, actor, is_admin):
|
||||
raise HTTPException(status_code=404, detail="归纳任务不存在")
|
||||
if not await _store(request).cancel_job(job_id):
|
||||
raise HTTPException(status_code=404, detail="归纳任务不存在")
|
||||
return {"canceled": True}
|
||||
|
||||
|
||||
@router.post("/sync-items/{item_id}/retry", status_code=202)
|
||||
async def retry_sync_item(request: Request, item_id: str) -> dict[str, Any]:
|
||||
actor, is_admin = await _actor(request)
|
||||
jobs = await _store(request).list_jobs(limit=200, created_by=None if is_admin else actor)
|
||||
item = next(
|
||||
(item for job in jobs for item in job.get("items", []) if item.get("id") == item_id),
|
||||
None,
|
||||
)
|
||||
if item is None:
|
||||
raise HTTPException(status_code=404, detail="归纳任务项不存在")
|
||||
if item.get("target_type") == "weknora":
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
str(item["target_id"]), actor, write=True, is_admin=is_admin
|
||||
)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="目标知识库不存在或当前用户不可写")
|
||||
row = await _store(request).requeue_item(item_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="归纳任务项不存在")
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/resync", status_code=202)
|
||||
async def resync_binding(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
actor, is_admin = await _actor(request)
|
||||
binding = await _store(request).get_binding(binding_id)
|
||||
if not _owned_by_actor(binding, actor, is_admin) or not binding.get("active"):
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
if binding.get("target_type") == "weknora":
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
str(binding["target_id"]), actor, write=True, is_admin=is_admin
|
||||
)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="目标知识库不存在或当前用户不可写")
|
||||
target = {
|
||||
"target_type": binding["target_type"],
|
||||
"target_mode": binding["target_mode"],
|
||||
"target_id": binding["target_id"],
|
||||
"target_name": binding["target_name"],
|
||||
"remote_business_key": binding.get("remote_business_key"),
|
||||
}
|
||||
job = await _store(request).create_job(
|
||||
skill_names=[binding["skill_name"]],
|
||||
targets=[target],
|
||||
created_by=actor,
|
||||
force=True,
|
||||
review_mode="auto_high_confidence",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/detach")
|
||||
async def detach_binding(request: Request, binding_id: str, body: DetachRequest) -> dict[str, Any]:
|
||||
actor, is_admin = await _actor(request)
|
||||
store = _store(request)
|
||||
binding = await store.get_binding(binding_id)
|
||||
if not _owned_by_actor(binding, actor, is_admin):
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
if await store.has_active_item(binding_id):
|
||||
raise HTTPException(status_code=409, detail="该绑定仍有运行中的归纳项,请先取消任务")
|
||||
cleanup_failed = False
|
||||
if body.delete_contributions:
|
||||
try:
|
||||
if binding["target_type"] == "assistant":
|
||||
sources = await request.app.state.assistant_knowledge_store.list_sources()
|
||||
for source in sources:
|
||||
if source.get("source_type") == "skill" and source.get("source_key") == binding["skill_name"]:
|
||||
await request.app.state.assistant_knowledge_store.delete_source(source["id"])
|
||||
elif binding["target_type"] == "weknora":
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
str(binding["target_id"]),
|
||||
actor,
|
||||
write=True,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if mapping is None or not runtime.weknora_enabled:
|
||||
raise RuntimeError("远端普通知识库当前不可写")
|
||||
client = build_weknora_client(runtime)
|
||||
remote_kb_id = str(mapping["weknora_id"])
|
||||
for artifact in await store.list_artifacts(binding_id):
|
||||
if artifact["kind"] == "wiki_page" and artifact.get("slug"):
|
||||
await client.delete_wiki_page(remote_kb_id, artifact["slug"])
|
||||
elif artifact.get("remote_id") and artifact["kind"] in {
|
||||
"raw_source",
|
||||
"summary_document",
|
||||
"faq_projection",
|
||||
"conversation_archive",
|
||||
"graph_projection",
|
||||
}:
|
||||
await client.delete_document(str(artifact["remote_id"]))
|
||||
except Exception: # noqa: BLE001
|
||||
cleanup_failed = True
|
||||
row = await store.detach_binding(
|
||||
binding_id,
|
||||
delete_contributions=body.delete_contributions,
|
||||
cleanup_failed=cleanup_failed,
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/reviews")
|
||||
async def list_reviews(
|
||||
request: Request,
|
||||
job_id: str | None = Query(default=None),
|
||||
item_id: str | None = Query(default=None),
|
||||
status: str | None = Query(default="pending"),
|
||||
) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
return {"reviews": await _store(request).list_reviews(job_id=job_id, item_id=item_id, status=status)}
|
||||
|
||||
|
||||
@router.post("/reviews/actions")
|
||||
async def review_actions(request: Request, body: ReviewRequest) -> dict[str, Any]:
|
||||
actor = await _require_admin(request)
|
||||
item_ids = await _store(request).review(body.review_ids, action=body.action, actor=actor, edits=body.edits, reason=body.reason)
|
||||
resumed: list[str] = []
|
||||
executor = request.app.state.skill_knowledge_executor
|
||||
for item_id in item_ids:
|
||||
if await _store(request).pending_review_count(item_id) == 0:
|
||||
asyncio.create_task(executor.resume_reviewed_item(item_id))
|
||||
resumed.append(item_id)
|
||||
return {"updated": len(body.review_ids), "resumed_item_ids": resumed}
|
||||
@@ -48,6 +48,14 @@ class PromptPrefixResponse(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeBaseProductVersionResponse(BaseModel):
|
||||
version: Literal["v1", "v2"] = "v1"
|
||||
|
||||
|
||||
class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
|
||||
version: Literal["v1", "v2"]
|
||||
|
||||
|
||||
class PromptPrefixUpdateRequest(BaseModel):
|
||||
enabled: bool | None = None
|
||||
switch_label: str | None = None
|
||||
@@ -175,6 +183,21 @@ async def update_prompt_prefix(body: PromptPrefixUpdateRequest, request: Request
|
||||
return _to_response(saved.prompt_prefix)
|
||||
|
||||
|
||||
@router.get("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
|
||||
async def get_knowledge_base_product_version() -> KnowledgeBaseProductVersionResponse:
|
||||
return KnowledgeBaseProductVersionResponse(version=load_system_settings().knowledge_base_product_version)
|
||||
|
||||
|
||||
@router.put("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
|
||||
async def update_knowledge_base_product_version(
|
||||
body: KnowledgeBaseProductVersionUpdateRequest,
|
||||
request: Request,
|
||||
) -> KnowledgeBaseProductVersionResponse:
|
||||
await _require_admin(request)
|
||||
saved = save_system_settings(load_system_settings().model_copy(update={"knowledge_base_product_version": body.version}))
|
||||
return KnowledgeBaseProductVersionResponse(version=saved.knowledge_base_product_version)
|
||||
|
||||
|
||||
@router.get("/plugin-management", response_model=PluginManagementResponse)
|
||||
async def get_plugin_management() -> PluginManagementResponse:
|
||||
"""Return the iframe plugin domain -> agent mappings."""
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Asynchronous, file-backed assistant export and ordinary Wiki restore."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||
from app.gateway.routers.assistant_knowledge import _actor, _list_export_manifests, _package_root, _read_export_manifest, _store, _write_export_manifest
|
||||
from deerflow.assistant_knowledge.archive import PackageArchive
|
||||
from deerflow.assistant_knowledge.package_io import package_path, write_jsonl_record
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.assistant_knowledge.store import _vector_blob_from_payload
|
||||
|
||||
router = APIRouter(tags=["wiki-packages"])
|
||||
|
||||
|
||||
def job_view(row):
|
||||
if row["mapping_id"].startswith("assistant:"):
|
||||
base_id = row["mapping_id"].split(":", 1)[1]
|
||||
row["package_download_url"] = f"/api/assistant-knowledge/bases/{base_id}/export-package/{row['id']}/download"
|
||||
elif row["mapping_id"].startswith("restore:"):
|
||||
row["package_download_url"] = None
|
||||
return row
|
||||
|
||||
|
||||
async def export_assistant(app, base_id, job_id):
|
||||
scope = f"assistant:{base_id}"
|
||||
path = package_path(_package_root(), job_id)
|
||||
temp = path.with_suffix(".tmp")
|
||||
counts = {}
|
||||
try:
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="exporting")
|
||||
with temp.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
async for kind, payload in app.state.assistant_knowledge_store.iter_export_records(base_id):
|
||||
await asyncio.to_thread(write_jsonl_record, handle, kind, payload)
|
||||
counts[kind] = counts.get(kind, 0) + 1
|
||||
if counts.get("wiki_page") and not counts.get("vector"):
|
||||
raise ValueError("该助手知识库尚无可导出的 Wiki 向量,请先从已向量化的普通知识库同步")
|
||||
await asyncio.to_thread(temp.replace, path)
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="completed", phase="package_ready", counts=counts)
|
||||
except Exception as exc:
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="failed", counts=counts, error=str(exc))
|
||||
|
||||
|
||||
@router.post("/api/assistant-knowledge/bases/{base_id}/export-package", status_code=202)
|
||||
async def create_export(request: Request, base_id: str):
|
||||
await _actor(request, admin=True)
|
||||
if not await _store(request).get_base(base_id):
|
||||
raise HTTPException(404, "助手知识库不存在")
|
||||
job_id = str(uuid4())
|
||||
row = _write_export_manifest(job_id=job_id, mapping_id=f"assistant:{base_id}", status="queued", phase="queued")
|
||||
spawn_local_wiki_index_job(request.app, export_assistant(request.app, base_id, job_id))
|
||||
return job_view(row)
|
||||
|
||||
|
||||
@router.get("/api/assistant-knowledge/bases/{base_id}/export-package")
|
||||
async def list_exports(request: Request, base_id: str):
|
||||
await _actor(request, admin=True)
|
||||
return {"jobs": [job_view(row) for row in await asyncio.to_thread(_list_export_manifests, f"assistant:{base_id}", limit=100)]}
|
||||
|
||||
|
||||
@router.get("/api/assistant-knowledge/bases/{base_id}/export-package/{job_id}/download")
|
||||
async def download_export(request: Request, base_id: str, job_id: str):
|
||||
await _actor(request, admin=True)
|
||||
row = _read_export_manifest(job_id)
|
||||
if not row or row["mapping_id"] != f"assistant:{base_id}" or row["status"] != "completed" or not row["file_exists"]:
|
||||
raise HTTPException(404, "导出包尚未就绪")
|
||||
return FileResponse(package_path(_package_root(), job_id), media_type="application/x-ndjson", filename=f"wiki-vector-{job_id}.jsonl")
|
||||
|
||||
|
||||
async def restore_package(app, mapping, job_id, path: Path):
|
||||
scope = f"restore:{mapping['id']}"
|
||||
archive = None
|
||||
counts = {"wiki_page": 0, "vector": 0}
|
||||
store = app.state.llmwiki_index_store
|
||||
owner = f"package:{job_id}"
|
||||
leased = False
|
||||
try:
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="validating")
|
||||
archive = await asyncio.to_thread(PackageArchive, path)
|
||||
embedding = app.state.llmwiki_embedding
|
||||
if embedding is None:
|
||||
raise ValueError("请配置与数据包相同的 Wiki 编码模型,以便复用向量进行检索")
|
||||
if embedding.dimensions is None:
|
||||
await embedding.embed_query("Wiki embedding dimension probe")
|
||||
# Validate the complete package before the first remote mutation.
|
||||
slugs = {p["slug"] for p in archive["wiki_pages"]}
|
||||
vector_slugs = set()
|
||||
for raw in archive["vectors"]:
|
||||
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "")
|
||||
if slug not in slugs or _vector_blob_from_payload(raw) is None:
|
||||
raise ValueError(f"向量数据或 Wiki 引用无效:{slug}")
|
||||
if raw.get("embedding_fingerprint") != embedding.fingerprint or raw.get("embedding_dimensions") != embedding.dimensions:
|
||||
raise ValueError("数据包编码模型指纹与当前配置不一致;请使用同一模型配置,禁止静默重新编码")
|
||||
vector_slugs.add(slug)
|
||||
if not vector_slugs:
|
||||
raise ValueError("数据包中没有 Wiki 向量")
|
||||
for page in archive["wiki_pages"]:
|
||||
if page["slug"] not in vector_slugs and page.get("page_type") not in {"index", "directory"} and (page.get("content") or "").strip():
|
||||
raise ValueError(f"Wiki 缺少向量:{page['slug']}")
|
||||
leased = await store.try_acquire_sync_lease(mapping["id"], owner=owner, sync_id=job_id, lease_seconds=900, fingerprint=embedding.fingerprint)
|
||||
if not leased:
|
||||
raise ValueError("目标知识库正在同步,请稍后重试")
|
||||
client = build_weknora_client(get_resolved_llmwiki_runtime(app.state.config))
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="restoring")
|
||||
folders = {(): ""}
|
||||
for raw in archive["wiki_pages"]:
|
||||
if not await store.renew_sync_lease(mapping["id"], owner=owner, lease_seconds=900):
|
||||
raise RuntimeError("导入租约已失效,请重新发起")
|
||||
# Restore the provider's Wiki schema, never raw documents; raw
|
||||
# uploads would trigger generation and embedding a second time.
|
||||
categories = raw.get("category_path") or []
|
||||
if isinstance(categories, str):
|
||||
categories = [part for part in categories.split("/") if part]
|
||||
parent = ()
|
||||
for name in categories:
|
||||
path_key = (*parent, name)
|
||||
if path_key not in folders:
|
||||
folder = await client.create_wiki_folder(mapping["weknora_id"], name=name, parent_id=folders[parent])
|
||||
folders[path_key] = folder["id"]
|
||||
parent = path_key
|
||||
payload = {key: value for key, value in raw.items() if key not in {"id", "tenant_id", "knowledge_base_id", "folder_id", "origins"}}
|
||||
payload["folder_id"] = folders[parent]
|
||||
if raw.get("tags"):
|
||||
payload["page_metadata"] = {**(raw.get("page_metadata") or {}), "tags": raw["tags"]}
|
||||
remote = await client.create_wiki_page(mapping["weknora_id"], payload)
|
||||
page = normalize_wiki_page(mapping["id"], {**raw, **remote})
|
||||
vectors = []
|
||||
for value in archive.vectors_for(raw["slug"]):
|
||||
blob, dimensions = _vector_blob_from_payload(value)
|
||||
vectors.append({**value, "vector_blob": blob, "embedding_dimensions": dimensions})
|
||||
await store.replace_page_vectors(page=page, vectors=vectors, fingerprint=embedding.fingerprint, sync_id=job_id)
|
||||
counts["wiki_page"] += 1
|
||||
counts["vector"] += len(vectors)
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="restoring", counts=counts)
|
||||
await client.rebuild_wiki_links(mapping["weknora_id"])
|
||||
await store.finish_sync(mapping["id"], owner=owner, sync_id=job_id, success=True, remote_page_count=counts["wiki_page"])
|
||||
leased = False
|
||||
if app.state.llmwiki_vector_cache:
|
||||
app.state.llmwiki_vector_cache.invalidate(mapping["id"])
|
||||
from app.gateway.knowledge_transfer import sync_to_global
|
||||
|
||||
await sync_to_global(app, mapping)
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="completed", phase="completed", counts=counts)
|
||||
except Exception as exc:
|
||||
if leased:
|
||||
await store.finish_sync(mapping["id"], owner=owner, sync_id=job_id, success=False, remote_page_count=counts["wiki_page"], error=str(exc))
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="failed", counts=counts, error=str(exc))
|
||||
finally:
|
||||
if archive is not None:
|
||||
await asyncio.to_thread(archive.close)
|
||||
|
||||
|
||||
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/import-package", status_code=202)
|
||||
async def upload_restore(request: Request, mapping_id: str, file: UploadFile = File(...)):
|
||||
actor = await _actor(request, admin=True)
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, actor or "system", write=True, is_admin=True)
|
||||
if not mapping or mapping.get("is_conversation_deposit") or mapping.get("owner_user_id") == "system":
|
||||
raise HTTPException(404, "目标普通知识库不存在或不可导入")
|
||||
client = build_weknora_client(get_resolved_llmwiki_runtime(request.app.state.config))
|
||||
remote = await client.get_knowledge_base(mapping["weknora_id"])
|
||||
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
|
||||
raise HTTPException(422, "请先开启目标知识库的 Wiki")
|
||||
existing = await client.list_wiki_pages(mapping["weknora_id"], page_size=1)
|
||||
if existing.get("total"):
|
||||
raise HTTPException(409, "请导入到新建的空 Wiki 知识库,避免覆盖已有页面")
|
||||
job_id = str(uuid4())
|
||||
scope = f"restore:{mapping_id}"
|
||||
row = _write_export_manifest(job_id=job_id, mapping_id=scope, status="queued", phase="uploading")
|
||||
path = package_path(_package_root(), job_id)
|
||||
try:
|
||||
with path.open("wb") as handle:
|
||||
while data := await file.read(1024 * 1024):
|
||||
await asyncio.to_thread(handle.write, data)
|
||||
except Exception as exc:
|
||||
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="upload_failed", error=str(exc))
|
||||
raise
|
||||
finally:
|
||||
await file.close()
|
||||
spawn_local_wiki_index_job(request.app, restore_package(request.app, mapping, job_id, path))
|
||||
return job_view(row)
|
||||
|
||||
|
||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/import-package")
|
||||
async def list_restores(request: Request, mapping_id: str):
|
||||
await _actor(request, admin=True)
|
||||
return {"jobs": [job_view(row) for row in await asyncio.to_thread(_list_export_manifests, f"restore:{mapping_id}", limit=100)]}
|
||||
@@ -405,6 +405,7 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
|
||||
"reasoning_effort",
|
||||
"rag_mode_enabled",
|
||||
"llmwiki_knowledge_base_ids",
|
||||
"assistant_knowledge_base_ids",
|
||||
"writing_mode",
|
||||
"writing_artifact_path",
|
||||
"canvas_artifact_title",
|
||||
@@ -1116,7 +1117,7 @@ async def start_run(
|
||||
if prefix_settings.enabled and prefix_settings.prompt_prefix.strip():
|
||||
graph_input = _prepend_prompt_prefix(graph_input, prefix_settings.prompt_prefix.strip())
|
||||
llmwiki_rag = (
|
||||
LlmWikiRagContext([], 0, "", [], "")
|
||||
LlmWikiRagContext([], [], 0, "", [], "")
|
||||
if fixed_answer is not None
|
||||
else await build_llmwiki_rag_context(
|
||||
request=request,
|
||||
@@ -1128,12 +1129,14 @@ async def start_run(
|
||||
graph_input = attach_llmwiki_rag_to_graph_input(graph_input, llmwiki_rag)
|
||||
effective_rag_mode = True
|
||||
run_metadata["llmwiki_knowledge_base_ids"] = llmwiki_rag.knowledge_base_ids
|
||||
run_metadata["assistant_knowledge_base_ids"] = llmwiki_rag.assistant_knowledge_base_ids
|
||||
run_metadata["llmwiki_rag_source_count"] = llmwiki_rag.source_count
|
||||
try:
|
||||
await run_ctx.thread_store.update_metadata(
|
||||
thread_id,
|
||||
{
|
||||
"llmwiki_knowledge_base_ids": llmwiki_rag.knowledge_base_ids,
|
||||
"assistant_knowledge_base_ids": llmwiki_rag.assistant_knowledge_base_ids,
|
||||
"rag_mode_enabled": True,
|
||||
"rag_mode": True,
|
||||
},
|
||||
@@ -1153,6 +1156,8 @@ async def start_run(
|
||||
body_context["rag_mode_enabled"] = True
|
||||
if "llmwiki_knowledge_base_ids" not in body_context and llmwiki_rag.knowledge_base_ids:
|
||||
body_context["llmwiki_knowledge_base_ids"] = llmwiki_rag.knowledge_base_ids
|
||||
if "assistant_knowledge_base_ids" not in body_context and llmwiki_rag.assistant_knowledge_base_ids:
|
||||
body_context["assistant_knowledge_base_ids"] = llmwiki_rag.assistant_knowledge_base_ids
|
||||
if not llmwiki_rag.knowledge_base_ids:
|
||||
existing_excluded = list(body_context.get("excluded_tools") or getattr(body, "excluded_tools", None) or [])
|
||||
if "llmwiki_search" not in existing_excluded:
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Debounced file watcher and cron scheduler for opted-in knowledge bindings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from croniter import croniter
|
||||
|
||||
from app.gateway.skill_knowledge_job_executor import _skill_directory
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.skill_knowledge import scan_skill
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillKnowledgeAutoSyncService:
|
||||
def __init__(self, app: Any, *, poll_seconds: float = 10.0) -> None:
|
||||
self.app = app
|
||||
self.store = app.state.skill_knowledge_store
|
||||
self.poll_seconds = poll_seconds
|
||||
self._task: asyncio.Task | None = None
|
||||
self._wake = asyncio.Event()
|
||||
self._observed: dict[str, tuple[str, datetime]] = {}
|
||||
self._schedule_expression = ""
|
||||
self._next_schedule: datetime | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._loop())
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _queue_skill(
|
||||
self,
|
||||
skill_name: str,
|
||||
bindings: list[dict[str, Any]],
|
||||
*,
|
||||
trigger: str,
|
||||
review_mode: str,
|
||||
) -> None:
|
||||
targets = [
|
||||
{
|
||||
"target_type": binding["target_type"],
|
||||
"target_mode": binding["target_mode"],
|
||||
"target_id": binding["target_id"],
|
||||
"target_name": binding["target_name"],
|
||||
"remote_business_key": binding.get("remote_business_key"),
|
||||
}
|
||||
for binding in bindings
|
||||
]
|
||||
job = await self.store.create_job(
|
||||
skill_names=[skill_name],
|
||||
targets=targets,
|
||||
created_by=None,
|
||||
trigger=trigger,
|
||||
review_mode=review_mode,
|
||||
)
|
||||
if job["total_count"]:
|
||||
self.app.state.skill_knowledge_dispatcher.nudge()
|
||||
|
||||
async def _watch(
|
||||
self,
|
||||
bindings_by_skill: dict[str, list[dict[str, Any]]],
|
||||
*,
|
||||
debounce_seconds: int,
|
||||
write_remote_enabled: bool,
|
||||
review_mode: str,
|
||||
) -> None:
|
||||
now = datetime.now(UTC)
|
||||
for skill_name, bindings in bindings_by_skill.items():
|
||||
try:
|
||||
root = _skill_directory(self.app.state.config, skill_name)
|
||||
scan = await asyncio.to_thread(scan_skill, skill_name, root)
|
||||
except Exception:
|
||||
logger.exception("Failed to fingerprint skill %s", skill_name)
|
||||
continue
|
||||
synced = {str(binding.get("synced_digest") or "") for binding in bindings}
|
||||
if synced == {scan.source_digest}:
|
||||
self._observed.pop(skill_name, None)
|
||||
continue
|
||||
await self.store.mark_skill_stale(skill_name)
|
||||
previous = self._observed.get(skill_name)
|
||||
if previous is None or previous[0] != scan.source_digest:
|
||||
self._observed[skill_name] = (scan.source_digest, now)
|
||||
continue
|
||||
if (now - previous[1]).total_seconds() < debounce_seconds:
|
||||
continue
|
||||
if write_remote_enabled:
|
||||
current = await self.store.list_bindings(skill_names=[skill_name])
|
||||
stale = [binding for binding in current if binding.get("auto_sync_enabled") and binding.get("status") == "stale"]
|
||||
if stale:
|
||||
await self._queue_skill(
|
||||
skill_name,
|
||||
stale,
|
||||
trigger="watcher",
|
||||
review_mode=review_mode,
|
||||
)
|
||||
self._observed.pop(skill_name, None)
|
||||
|
||||
def _schedule_due(self, expression: str, now: datetime) -> bool:
|
||||
if expression != self._schedule_expression or self._next_schedule is None:
|
||||
self._schedule_expression = expression
|
||||
self._next_schedule = croniter(expression, now).get_next(datetime)
|
||||
return False
|
||||
if now < self._next_schedule:
|
||||
return False
|
||||
self._next_schedule = croniter(expression, now).get_next(datetime)
|
||||
return True
|
||||
|
||||
async def _tick(self) -> None:
|
||||
settings = (await asyncio.to_thread(load_system_settings)).skill_knowledge_auto_sync
|
||||
bindings = [row for row in await self.store.list_bindings() if row.get("auto_sync_enabled")]
|
||||
by_skill: dict[str, list[dict[str, Any]]] = {}
|
||||
for binding in bindings:
|
||||
by_skill.setdefault(str(binding["skill_name"]), []).append(binding)
|
||||
if settings.watch_enabled:
|
||||
await self._watch(
|
||||
by_skill,
|
||||
debounce_seconds=settings.debounce_seconds,
|
||||
write_remote_enabled=settings.write_remote_enabled,
|
||||
review_mode=settings.review_mode,
|
||||
)
|
||||
if settings.schedule_enabled and settings.write_remote_enabled and by_skill and self._schedule_due(settings.schedule_cron, datetime.now(UTC)):
|
||||
for skill_name, skill_bindings in by_skill.items():
|
||||
await self._queue_skill(
|
||||
skill_name,
|
||||
skill_bindings,
|
||||
trigger="scheduled",
|
||||
review_mode=settings.review_mode,
|
||||
)
|
||||
|
||||
async def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick()
|
||||
except Exception:
|
||||
logger.exception("Skill knowledge automatic sync tick failed")
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=self.poll_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Small persistent-queue dispatcher for skill knowledge items."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillKnowledgeJobDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
executor: Any,
|
||||
*,
|
||||
interval_seconds: float = 2.0,
|
||||
concurrency: int = 3,
|
||||
lease_seconds: int = 900,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.executor = executor
|
||||
self.interval_seconds = interval_seconds
|
||||
self.lease_seconds = lease_seconds
|
||||
self.owner = f"{socket.gethostname()}:{uuid4()}"
|
||||
self.semaphore = asyncio.Semaphore(concurrency)
|
||||
self._wake = asyncio.Event()
|
||||
self._task: asyncio.Task | None = None
|
||||
self._running: set[str] = set()
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._loop())
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _run(self, item: dict) -> None:
|
||||
item_id = str(item["id"])
|
||||
async with self.semaphore:
|
||||
heartbeat = asyncio.create_task(self._heartbeat(item_id))
|
||||
try:
|
||||
await self.executor.run_item(item)
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
try:
|
||||
await heartbeat
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._running.discard(item_id)
|
||||
self._wake.set()
|
||||
|
||||
async def _heartbeat(self, item_id: str) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(min(30, max(5, self.lease_seconds // 3)))
|
||||
if not await self.store.renew_lease(
|
||||
item_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.lease_seconds,
|
||||
):
|
||||
return
|
||||
|
||||
async def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
for item in await self.store.list_queued_items(limit=20):
|
||||
item_id = str(item["id"])
|
||||
if item_id in self._running:
|
||||
continue
|
||||
claimed = await self.store.claim_item(
|
||||
item_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.lease_seconds,
|
||||
)
|
||||
if claimed is None:
|
||||
continue
|
||||
self._running.add(item_id)
|
||||
asyncio.create_task(self._run(claimed))
|
||||
except Exception:
|
||||
logger.exception("Skill knowledge dispatcher tick failed")
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=self.interval_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Background executor for one skill x target matrix item."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from deerflow.assistant_knowledge import export_package_from_skill_ir
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
|
||||
from deerflow.skill_knowledge.graph_writer import parameterized_statements, plan_from_ir
|
||||
from deerflow.skills.parser import parse_skill_file
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.types import SkillCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _skill_directory(app_config: Any, skill_name: str) -> Path:
|
||||
storage = get_or_new_skill_storage(app_config=app_config)
|
||||
root = storage.get_skills_root_path().resolve()
|
||||
for category in SkillCategory:
|
||||
direct = root / category.value / skill_name
|
||||
if direct.resolve().is_relative_to((root / category.value).resolve()) and (direct / "SKILL.md").is_file():
|
||||
return direct
|
||||
for category in SkillCategory:
|
||||
category_root = root / category.value
|
||||
if not category_root.is_dir():
|
||||
continue
|
||||
for skill_md in category_root.rglob("SKILL.md"):
|
||||
if skill_md.is_symlink():
|
||||
continue
|
||||
parsed = parse_skill_file(skill_md, category=category, relative_path=skill_md.parent.relative_to(category_root))
|
||||
if parsed and parsed.name == skill_name:
|
||||
return skill_md.parent
|
||||
raise FileNotFoundError(f"Skill '{skill_name}' not found under running skill roots")
|
||||
|
||||
|
||||
def _graph_projection(ir: dict[str, Any]) -> str:
|
||||
rows = []
|
||||
entities = {str(item.get("local_id")): item for item in ir.get("entities", [])}
|
||||
for relation in ir.get("relations", []):
|
||||
if float(relation.get("confidence") or 0) < 0.75 or not relation.get("evidence"):
|
||||
continue
|
||||
source = entities.get(str(relation.get("source_local_id")), {})
|
||||
target = entities.get(str(relation.get("target_local_id")), {})
|
||||
rows.append(
|
||||
json.dumps(
|
||||
{
|
||||
"source": source.get("name"),
|
||||
"source_type": source.get("type"),
|
||||
"predicate": relation.get("predicate"),
|
||||
"target": target.get("name"),
|
||||
"target_type": target.get("type"),
|
||||
"description": relation.get("description"),
|
||||
"confidence": relation.get("confidence"),
|
||||
"evidence": relation.get("evidence"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
class SkillKnowledgeJobExecutor:
|
||||
def __init__(self, app: Any) -> None:
|
||||
self.app = app
|
||||
self.store = app.state.skill_knowledge_store
|
||||
|
||||
async def _cancel_if_requested(self, item: dict[str, Any]) -> bool:
|
||||
job = await self.store.get_job(str(item["job_id"]))
|
||||
if not (job or {}).get("cancel_requested"):
|
||||
return False
|
||||
await self.store.update_item(
|
||||
str(item["id"]),
|
||||
status="canceled",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
)
|
||||
return True
|
||||
|
||||
async def _assert_source_unchanged(self, item: dict[str, Any], expected_digest: str) -> None:
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
current = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
current = await enrich_converted_documents(current)
|
||||
if current.source_digest == expected_digest:
|
||||
return
|
||||
await self.store.mark_skill_stale(str(item["skill_name"]))
|
||||
raise RuntimeError("技能知识文件在归纳期间发生变化,旧版本保持激活,请重试")
|
||||
|
||||
async def _write_direct_graph(
|
||||
self,
|
||||
item: dict[str, Any],
|
||||
snapshot_id: str,
|
||||
ir: dict[str, Any],
|
||||
) -> dict[str, Any] | None:
|
||||
config = self.app.state.config.llmwiki.weknora.graph
|
||||
if not config.direct_write_enabled:
|
||||
return None
|
||||
if not config.neo4j_http_url or not config.username or not config.password:
|
||||
raise RuntimeError("Neo4j direct writer 已开启但后端连接配置不完整")
|
||||
plan = plan_from_ir(
|
||||
ir,
|
||||
target_type=str(item["target_type"]),
|
||||
target_id=str(item["target_id"]),
|
||||
binding_id=str(item["binding_id"]),
|
||||
snapshot_id=snapshot_id,
|
||||
job_id=str(item["job_id"]),
|
||||
)
|
||||
statements = parameterized_statements(plan)
|
||||
if not statements:
|
||||
return {"nodes": 0, "edges": 0, "statement_count": 0}
|
||||
endpoint = f"{config.neo4j_http_url.rstrip('/')}/db/{config.database}/tx/commit"
|
||||
async with httpx.AsyncClient(
|
||||
auth=(config.username, config.password),
|
||||
timeout=config.timeout_seconds,
|
||||
) as client:
|
||||
response = await client.post(endpoint, json={"statements": statements})
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
errors = payload.get("errors") or []
|
||||
if errors:
|
||||
raise RuntimeError(f"Neo4j transaction failed: {str(errors[0].get('code') or 'unknown')}")
|
||||
return {
|
||||
"nodes": len(plan.nodes),
|
||||
"edges": len(plan.edges),
|
||||
"statement_count": len(statements),
|
||||
"scope": plan.scope,
|
||||
}
|
||||
|
||||
async def run_item(self, item: dict[str, Any]) -> None:
|
||||
item_id = str(item["id"])
|
||||
try:
|
||||
await self.store.update_item(item_id, status="running", phase="scanning", progress=5)
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
await self.store.update_item(item_id, phase="extracting", progress=30)
|
||||
skill_record = await self.app.state.skill_store.get_any(str(item["skill_name"]))
|
||||
ir = await asyncio.to_thread(
|
||||
build_skill_ir,
|
||||
scan,
|
||||
display_name=(skill_record or {}).get("name_zh"),
|
||||
description=str((skill_record or {}).get("description") or ""),
|
||||
)
|
||||
snapshot = await self.store.get_or_create_snapshot(
|
||||
skill_name=str(item["skill_name"]),
|
||||
source_digest=scan.source_digest,
|
||||
extractor_version=str(ir["extractor_version"]),
|
||||
prompt_hash=str(ir["prompt_hash"]),
|
||||
model_name=str(ir["model_name"]),
|
||||
manifest=scan.manifest,
|
||||
ir=ir,
|
||||
counts=scan.counts,
|
||||
)
|
||||
snapshot_id = str(snapshot["id"])
|
||||
await self.store.update_item(item_id, phase="awaiting_review", progress=45, snapshot_id=snapshot_id)
|
||||
job = await self.store.get_job(str(item["job_id"]))
|
||||
review_mode = str((job or {}).get("review_mode") or "auto_high_confidence")
|
||||
review_objects: list[tuple[str, dict]] = []
|
||||
if review_mode == "required":
|
||||
review_objects = [("entity", obj) for obj in ir.get("entities", [])] + [("relation", obj) for obj in ir.get("relations", [])]
|
||||
elif review_mode == "auto_high_confidence":
|
||||
review_objects = [(kind, obj) for kind, rows in (("entity", ir.get("entities", [])), ("relation", ir.get("relations", []))) for obj in rows if float(obj.get("confidence") or 0) < 0.9]
|
||||
if review_objects:
|
||||
existing = await self.store.list_reviews(item_id=item_id, status=None)
|
||||
if not existing:
|
||||
await self.store.create_reviews(
|
||||
job_id=str(item["job_id"]),
|
||||
item_id=item_id,
|
||||
binding_id=str(item["binding_id"]),
|
||||
snapshot_id=snapshot_id,
|
||||
objects=review_objects,
|
||||
)
|
||||
await self.store.update_item(item_id, status="awaiting_review", phase="awaiting_review", progress=50)
|
||||
return
|
||||
await self._project_and_activate(item, snapshot_id, ir, scan)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Skill knowledge item failed: item=%s", item_id)
|
||||
await self.store.update_item(
|
||||
item_id,
|
||||
status="failed",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
error_code=type(exc).__name__,
|
||||
error_message=str(exc)[:2000],
|
||||
)
|
||||
|
||||
async def resume_reviewed_item(self, item_id: str) -> None:
|
||||
if await self.store.pending_review_count(item_id):
|
||||
return
|
||||
jobs = await self.store.list_jobs(limit=100)
|
||||
item = None
|
||||
for job in jobs:
|
||||
detail = await self.store.get_job(str(job["id"]))
|
||||
item = next((row for row in (detail or {}).get("items", []) if row["id"] == item_id), None)
|
||||
if item:
|
||||
break
|
||||
if not item or not item.get("snapshot_id"):
|
||||
return
|
||||
snapshot = await self.store.get_snapshot(str(item["snapshot_id"]))
|
||||
ir = await self.store.effective_ir(str(item["snapshot_id"]), item_id)
|
||||
if not snapshot or not ir:
|
||||
return
|
||||
try:
|
||||
await self.store.update_item(item_id, status="running", phase="merging", progress=55)
|
||||
await self._project_and_activate(item, str(item["snapshot_id"]), ir, None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Reviewed skill knowledge item failed: item=%s", item_id)
|
||||
await self.store.update_item(
|
||||
item_id,
|
||||
status="failed",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
error_code=type(exc).__name__,
|
||||
error_message=str(exc)[:2000],
|
||||
)
|
||||
|
||||
async def _project_and_activate(self, item: dict[str, Any], snapshot_id: str, ir: dict[str, Any], scan: Any | None) -> None:
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
expected_digest = str(ir.get("skill", {}).get("source_digest") or "")
|
||||
if scan is None:
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
if item["target_type"] == "weknora" and item["target_mode"] == "wiki":
|
||||
await self._generate_vectorize_and_deposit(item, snapshot_id, ir, scan)
|
||||
return
|
||||
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
|
||||
target_type = str(item["target_type"])
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
binding_status = "synced"
|
||||
page = ir["wiki"]["skill_page"]
|
||||
if target_type == "assistant":
|
||||
package = export_package_from_skill_ir(ir)
|
||||
await self.app.state.assistant_knowledge_store.import_package(
|
||||
package=package,
|
||||
source_type="skill",
|
||||
source_key=str(item["skill_name"]),
|
||||
source_name=str(ir["skill"].get("display_name") or item["skill_name"]),
|
||||
trigger="skill_direct",
|
||||
created_by=(await self.store.get_job(str(item["job_id"]))).get("created_by"),
|
||||
skill_name=str(item["skill_name"]),
|
||||
)
|
||||
artifacts.append({"kind": "assistant_wiki_revision", "slug": page["slug"], "content": page["content"], "business_key": f"assistant:wiki:{page['slug']}"})
|
||||
elif str(item["target_mode"]) == "readonly_sidecar":
|
||||
artifacts.append({"kind": "sidecar_wiki", "slug": page["slug"], "content": page["content"], "business_key": f"sidecar:{item['target_id']}:{page['slug']}"})
|
||||
artifacts.append({"kind": "sidecar_graph", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "content": _graph_projection(ir)})
|
||||
binding_status = "pending_remote_write"
|
||||
else:
|
||||
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise ValueError("Target WeKnora mapping does not exist")
|
||||
runtime = get_resolved_llmwiki_runtime(self.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise RuntimeError("普通知识库服务未配置")
|
||||
client = build_weknora_client(runtime)
|
||||
remote_kb_id = str(mapping["weknora_id"])
|
||||
if scan is not None:
|
||||
for file in scan.files:
|
||||
source_path = Path(scan.source_root) / file.path
|
||||
if file.text:
|
||||
remote = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/source/{file.path}", content=file.text)
|
||||
else:
|
||||
remote = await client.upload_document(
|
||||
remote_kb_id,
|
||||
filename=f"{item['skill_name']}-{source_path.name}",
|
||||
content=await asyncio.to_thread(source_path.read_bytes),
|
||||
)
|
||||
remote_id = str(remote.get("id") or remote.get("knowledge_id") or "") or None
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "raw_source",
|
||||
"slug": file.path,
|
||||
"remote_id": remote_id,
|
||||
"business_key": f"weknora:document:{remote_id}" if remote_id else None,
|
||||
"remote_refs": [{"provider": "weknora", "remote_kind": "document", "remote_id": remote_id}] if remote_id else [],
|
||||
}
|
||||
)
|
||||
target_mode = str(item["target_mode"])
|
||||
if target_mode == "wiki":
|
||||
remote_page: dict[str, Any]
|
||||
try:
|
||||
await client.get_wiki_page(remote_kb_id, page["slug"])
|
||||
except WeKnoraError as exc:
|
||||
if exc.status_code != 404:
|
||||
raise
|
||||
remote_page = await client.create_wiki_page(
|
||||
remote_kb_id,
|
||||
{
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"content": page["content"],
|
||||
"summary": str(ir["skill"].get("description") or ""),
|
||||
"page_type": "skill",
|
||||
},
|
||||
)
|
||||
else:
|
||||
remote_page = await client.update_wiki_page(
|
||||
remote_kb_id,
|
||||
page["slug"],
|
||||
{
|
||||
"title": page["title"],
|
||||
"content": page["content"],
|
||||
"summary": str(ir["skill"].get("description") or ""),
|
||||
},
|
||||
)
|
||||
remote_page_id = str(remote_page.get("id") or remote_page.get("page_id") or page["slug"])
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"slug": page["slug"],
|
||||
"content": page["content"],
|
||||
"remote_id": remote_page_id,
|
||||
"business_key": f"weknora:wiki_page:{remote_page_id}",
|
||||
"remote_refs": [
|
||||
{
|
||||
"provider": "weknora",
|
||||
"remote_kind": "wiki_page",
|
||||
"remote_id": remote_page_id,
|
||||
"slug": page["slug"],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
else:
|
||||
document_content = page["content"]
|
||||
artifact_kind = "summary_document"
|
||||
if target_mode == "faq":
|
||||
artifact_kind = "faq_projection"
|
||||
document_content = f"问题:{page['title']} 能解决什么问题,如何使用?\n\n回答:\n{page['content']}"
|
||||
elif target_mode == "conversation_archive":
|
||||
artifact_kind = "conversation_archive"
|
||||
document_content = f"# 技能归纳会话归档:{page['title']}\n\n以下内容由技能知识归纳任务生成。\n\n{page['content']}"
|
||||
remote_document = await client.create_manual_document(
|
||||
remote_kb_id,
|
||||
title=f"skills/{item['skill_name']}/{target_mode}.md",
|
||||
content=document_content,
|
||||
)
|
||||
remote_document_id = str(remote_document.get("id") or remote_document.get("knowledge_id") or "") or None
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": artifact_kind,
|
||||
"slug": f"skills/{item['skill_name']}/{target_mode}.md",
|
||||
"content": document_content,
|
||||
"remote_id": remote_document_id,
|
||||
"business_key": (f"weknora:document:{remote_document_id}" if remote_document_id else None),
|
||||
"remote_refs": (
|
||||
[
|
||||
{
|
||||
"provider": "weknora",
|
||||
"remote_kind": "document",
|
||||
"remote_id": remote_document_id,
|
||||
}
|
||||
]
|
||||
if remote_document_id
|
||||
else []
|
||||
),
|
||||
}
|
||||
)
|
||||
projection = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/graph-projection.jsonl", content=_graph_projection(ir))
|
||||
projection_id = str(projection.get("id") or projection.get("knowledge_id") or "") or None
|
||||
artifacts.append({"kind": "graph_projection", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "remote_id": projection_id})
|
||||
try:
|
||||
graph_audit = await self._write_direct_graph(item, snapshot_id, ir)
|
||||
if graph_audit is not None:
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "neo4j_direct_write_audit",
|
||||
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
|
||||
"content": json.dumps(graph_audit, ensure_ascii=False),
|
||||
"metadata": graph_audit,
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Optional Neo4j direct graph write failed: item=%s", item["id"])
|
||||
binding_status = "partial_failed"
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "neo4j_direct_write_audit",
|
||||
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
|
||||
"content": json.dumps(
|
||||
{"status": "failed", "error": type(exc).__name__},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
"metadata": {
|
||||
"status": "failed",
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
}
|
||||
)
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
|
||||
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
|
||||
|
||||
async def _generate_vectorize_and_deposit(self, item, snapshot_id, ir, scan):
|
||||
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise ValueError("目标普通知识库不存在")
|
||||
sync = getattr(self.app.state, "llmwiki_sync_service", None)
|
||||
if sync is None:
|
||||
raise ValueError("请先配置 Wiki 编码模型并启用本地向量索引;归纳需要完成向量化后同步知识梳理")
|
||||
client = build_weknora_client(get_resolved_llmwiki_runtime(self.app.state.config))
|
||||
remote_id = mapping["weknora_id"]
|
||||
remote = await client.get_knowledge_base(remote_id)
|
||||
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
|
||||
raise ValueError("目标普通知识库尚未开启 Wiki")
|
||||
artifacts = []
|
||||
document_ids = []
|
||||
await self.store.update_item(str(item["id"]), phase="writing_sources", progress=55)
|
||||
for file in scan.files:
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
if not file.text:
|
||||
continue
|
||||
document = await client.create_manual_document(remote_id, title=f"{ir['skill'].get('display_name') or item['skill_name']} · {file.path}", content=file.text)
|
||||
doc_id = str(document.get("id") or document.get("knowledge_id") or "")
|
||||
if not doc_id:
|
||||
raise ValueError("资料上传未返回文档标识")
|
||||
document_ids.append(doc_id)
|
||||
artifacts.append({"kind": "raw_source", "slug": file.path, "remote_id": doc_id})
|
||||
if not document_ids:
|
||||
raise ValueError("该技能没有可归纳的知识正文,代码、脚本和代码块不参与归纳")
|
||||
await self.store.update_item(str(item["id"]), phase="generating_wiki", progress=65)
|
||||
# WeKnora performs model extraction, categorization and linking. Wait
|
||||
# for both parsing and Wiki's debounced finalize before vectorization.
|
||||
stable = 0
|
||||
for _ in range(720):
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
docs = [await client.get_document(doc_id) for doc_id in document_ids]
|
||||
if any(doc.get("parse_status") in {"failed", "error"} for doc in docs):
|
||||
raise RuntimeError("WeKnora 资料解析失败,请查看普通知识库中的文档状态")
|
||||
stats = await client.get_wiki_stats(remote_id)
|
||||
done = all(doc.get("parse_status") == "completed" for doc in docs) and not stats.get("is_active") and not stats.get("pending_tasks")
|
||||
stable = stable + 1 if done else 0
|
||||
if stable >= 3:
|
||||
if not int(stats.get("total_pages") or 0):
|
||||
raise RuntimeError("WeKnora 解析已结束,但未生成 Wiki;请检查 Wiki 生成模型的调用错误、余额及资料是否含实质知识,再重试")
|
||||
break
|
||||
await asyncio.sleep(5)
|
||||
else:
|
||||
raise TimeoutError("Wiki 分析尚未完成;请检查普通知识库的生成任务后重新归纳")
|
||||
await self.store.update_item(str(item["id"]), phase="vectorizing_wiki", progress=80)
|
||||
result = await sync.sync_mapping(mapping, force=False, notify=False)
|
||||
if result.get("status") != "completed":
|
||||
raise RuntimeError("Wiki 向量化未全部完成,请检查向量化进度")
|
||||
await self.store.update_item(str(item["id"]), phase="syncing_assistant", progress=92)
|
||||
from app.gateway.knowledge_transfer import sync_to_global
|
||||
|
||||
deposited = await sync_to_global(self.app, mapping)
|
||||
artifacts.append({"kind": "assistant_deposit", "slug": "knowledge-summary", "metadata": deposited})
|
||||
await self._assert_source_unchanged(item, str(ir["skill"]["source_digest"]))
|
||||
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status="synced")
|
||||
Reference in new issue
Block a user