feat: add artifact library and improve public wiki

This commit is contained in:
吉鑫升 committed 2026-09-01 18:18:28 +08:00
1 parent 1b43205c1b
commit dd0ee282df
16 files changed
+4081 -98

No files matched your search

@@ -55,6 +55,7 @@ from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.persistence.llmwiki import LlmWikiStore
@@ -496,15 +497,7 @@ def _should_follow_weknora_binary_redirect(path: str) -> bool:
"""Keep protected preview/image bytes inside the DeerFlow proxy."""
segments = _path_segments(path)
return (
len(segments) == 5
and segments[:3] == ["api", "v1", "knowledge"]
and segments[4] == "preview"
) or (
len(segments) == 5
and segments[:3] == ["api", "v1", "knowledge-bases"]
and segments[4] == "files"
)
return (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge"] and segments[4] == "preview") or (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge-bases"] and segments[4] == "files")
def _forward_request_headers(request: Request) -> dict[str, str]:
@@ -1833,6 +1826,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"knowledge_count": int(source.get("knowledge_count") or 0),
"chunk_count": int(source.get("chunk_count") or 0),
"processing_count": int(source.get("processing_count") or 0),
"share_count": int(source.get("share_count") or 0),
"is_processing": bool(source.get("is_processing") or int(source.get("processing_count") or 0) > 0),
"is_temporary": bool(source.get("is_temporary", False)),
"embedding_model_id": source.get("embedding_model_id"),
"vector_store_id": source.get("vector_store_id"),
"vector_store_name": source.get("vector_store_name"),
@@ -1840,16 +1836,23 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"vector_store_engine_type": source.get("vector_store_engine_type"),
"vector_store_status": source.get("vector_store_status"),
"chunking_config": source.get("chunking_config") if isinstance(source.get("chunking_config"), dict) else {},
"image_processing_config": (source.get("image_processing_config") if isinstance(source.get("image_processing_config"), dict) else {}),
"indexing_strategy": source.get("indexing_strategy") if isinstance(source.get("indexing_strategy"), dict) else {},
"capabilities": source.get("capabilities") if isinstance(source.get("capabilities"), dict) else {},
"extract_config": source.get("extract_config") if isinstance(source.get("extract_config"), dict) else {},
"faq_config": source.get("faq_config") if isinstance(source.get("faq_config"), dict) else {},
"wiki_config": source.get("wiki_config") if isinstance(source.get("wiki_config"), dict) else {},
"question_generation_config": (source.get("question_generation_config") if isinstance(source.get("question_generation_config"), dict) else {}),
"auto_tag_config": source.get("auto_tag_config") if isinstance(source.get("auto_tag_config"), dict) else {},
"vlm_config": source.get("vlm_config") if isinstance(source.get("vlm_config"), dict) else {},
"asr_config": source.get("asr_config") if isinstance(source.get("asr_config"), dict) else {},
"summary_model_id": source.get("summary_model_id"),
"storage_provider": (source.get("storage_provider_config", {}).get("provider") if isinstance(source.get("storage_provider_config"), dict) else None),
"created_at": row.get("created_at"),
"updated_at": row.get("updated_at"),
"published_at": row.get("published_at"),
"remote_created_at": source.get("created_at"),
"remote_updated_at": source.get("updated_at"),
}
@@ -1877,11 +1880,7 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
write=False,
is_admin=False,
)
if (
row is None
or row.get("publication_status") != "published"
or _is_conversation_deposit_mapping(row)
):
if row is None or row.get("publication_status") != "published" or _is_conversation_deposit_mapping(row):
raise HTTPException(status_code=404, detail="Published knowledge base not found")
return row
@@ -2537,24 +2536,260 @@ async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest)
return {"success": True, "provider": "weknora", "health": health}
_PUBLIC_WIKI_SUMMARY_CONCURRENCY = 8
_PUBLIC_WIKI_PAGE_SIZE = 500
_PUBLIC_WIKI_PAGE_LIMIT = 100
_GENERATED_PUBLIC_WIKI_INDEX_SLUG = "__index__"
def _is_public_wiki_index_page(page: dict[str, Any]) -> bool:
page_type = str(page.get("page_type") or "").strip().lower()
title = str(page.get("title") or "").strip().lower()
slug_name = str(page.get("slug") or "").strip("/").rsplit("/", 1)[-1].lower()
return page_type == "index" or title in {"索引", "index"} or slug_name in {"index", "home", "readme"}
def _public_wiki_default_page(pages: list[dict[str, Any]]) -> dict[str, Any] | None:
return (
next((page for page in pages if _is_public_wiki_index_page(page)), None)
or next((page for page in pages if str(page.get("page_type") or "").lower() == "overview"), None)
or next(
(page for page in pages if not str(page.get("parent_slug") or "").strip() and int(page.get("depth") or 0) == 0),
None,
)
or (pages[0] if pages else None)
)
def _count_public_wiki_values(pages: list[dict[str, Any]], key: str, fallback: str) -> dict[str, int]:
counts: dict[str, int] = {}
for page in pages:
value = str(page.get(key) or fallback).strip().lower() or fallback
counts[value] = counts.get(value, 0) + 1
return counts
def _public_wiki_time_bound(pages: list[dict[str, Any]], key: str, *, latest: bool) -> str | None:
values = [str(page.get(key) or "").strip() for page in pages]
values = [value for value in values if value]
if not values:
return None
return max(values) if latest else min(values)
async def _all_public_wiki_page_views(
client: WeKnoraClient,
remote_id: str,
) -> tuple[list[dict[str, Any]], int, bool]:
pages_by_slug: dict[str, dict[str, Any]] = {}
remote_total = 0
complete = False
for page_number in range(1, _PUBLIC_WIKI_PAGE_LIMIT + 1):
result = await client.list_wiki_pages(
remote_id,
page=page_number,
page_size=_PUBLIC_WIKI_PAGE_SIZE,
query="",
)
raw_pages = result.get("pages") if isinstance(result, dict) else []
page_items = raw_pages if isinstance(raw_pages, list) else []
for page_item in page_items:
if not isinstance(page_item, dict):
continue
page = _wiki_page_view(page_item)
if page is None:
continue
slug = str(page.get("slug") or "").strip()
pages_by_slug[slug or f"__page_{len(pages_by_slug)}"] = page
reported_total = int(result.get("total") or 0)
remote_total = max(remote_total, reported_total, len(pages_by_slug))
total_pages = max(1, int(result.get("total_pages") or 1))
if not page_items or page_number >= total_pages or (reported_total > 0 and len(pages_by_slug) >= remote_total):
complete = True
break
return list(pages_by_slug.values()), max(remote_total, len(pages_by_slug)), complete
def _public_wiki_summary(
mapping_id: str,
pages: list[dict[str, Any]],
*,
total: int,
complete: bool,
) -> dict[str, Any]:
explicit_index = next((page for page in pages if _is_public_wiki_index_page(page)), None)
default_page = _public_wiki_default_page(pages)
if explicit_index is not None:
index_mode = "explicit"
index_slug: str | None = str(explicit_index.get("slug") or "") or None
elif pages:
index_mode = "generated"
index_slug = _GENERATED_PUBLIC_WIKI_INDEX_SLUG
else:
index_mode = "none"
index_slug = None
default_slug = index_slug or (str(default_page.get("slug") or "") if default_page else None)
route = f"/embed/knowledge/{quote(mapping_id, safe='')}"
if default_slug:
route = f"{route}?{urlencode({'wikiSlug': default_slug})}"
return {
"status": "ready" if pages else "empty",
"page_count": total,
"loaded_page_count": len(pages),
"stats_complete": complete,
"page_type_counts": _count_public_wiki_values(pages, "page_type", "page"),
"page_status_counts": _count_public_wiki_values(pages, "status", "unknown"),
"index_mode": index_mode,
"has_explicit_index": explicit_index is not None,
"index_slug": index_slug,
"default_slug": default_slug,
"created_at": _public_wiki_time_bound(pages, "created_at", latest=False),
"updated_at": _public_wiki_time_bound(pages, "updated_at", latest=True),
"embed_route": route,
}
def _unavailable_public_wiki_summary(mapping_id: str) -> dict[str, Any]:
return {
"status": "unavailable",
"page_count": 0,
"loaded_page_count": 0,
"stats_complete": False,
"page_type_counts": {},
"page_status_counts": {},
"index_mode": "unknown",
"has_explicit_index": False,
"index_slug": None,
"default_slug": None,
"created_at": None,
"updated_at": None,
"embed_route": f"/embed/knowledge/{quote(mapping_id, safe='')}",
}
def _public_local_index_summary(
request: Request,
mapping: dict[str, Any],
state: dict[str, Any] | None,
) -> dict[str, Any]:
app_state = request.app.state
local_config = getattr(
getattr(getattr(app_state, "config", None), "llmwiki", None),
"local_wiki_index",
None,
)
enabled = bool(getattr(local_config, "enabled", False))
current = dict(state or {})
state_name = str(current.get("state") or "not_synced")
if state_name == "idle" and int(current.get("index_revision") or 0) == 0 and not current.get("last_completed_at") and not current.get("last_success_at"):
state_name = "not_synced"
embedding = getattr(app_state, "llmwiki_embedding", None)
fingerprint = embedding.fingerprint if embedding is not None else None
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(
current,
fingerprint,
)
return {
"enabled": enabled,
"wiki_index_enabled": bool(mapping.get("wiki_index_enabled", True)),
"state": state_name,
"fully_vectorized": fully_vectorized,
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
"remote_page_count": int(current.get("remote_page_count") or 0),
"local_page_count": int(current.get("local_page_count") or 0),
"ready_page_count": int(current.get("ready_page_count") or 0),
"failed_page_count": int(current.get("failed_page_count") or 0),
"vector_count": int(current.get("vector_count") or 0),
"last_started_at": current.get("last_started_at"),
"last_completed_at": current.get("last_completed_at"),
"last_success_at": current.get("last_success_at"),
"updated_at": current.get("updated_at"),
}
@router.get("/api/public/llmwiki/knowledge-bases")
async def list_public_knowledge_bases(request: Request) -> dict[str, Any]:
"""List the minimal identity of every knowledge base published as a public Wiki."""
"""List live, published Wiki libraries with safe metadata and index health."""
rows = await _store(request).list_visible(
"__public_wiki__",
scope="public",
is_admin=False,
)
knowledge_bases = [
{
"id": str(row["id"]),
"name": str(row.get("name") or ""),
}
for row in rows
if row.get("publication_status") == "published"
and not _is_conversation_deposit_mapping(row)
rows = [
row
for row in await _store(request).list_visible(
"__public_wiki__",
scope="public",
is_admin=False,
)
if row.get("publication_status") == "published" and not _is_conversation_deposit_mapping(row)
]
client = _client_or_503()
try:
remote_items = await client.list_knowledge_bases()
except WeKnoraError as exc:
_raise_upstream(exc)
remote_by_id = {str(item.get("id")): item for item in remote_items if isinstance(item, dict) and item.get("id")}
# A successful upstream list is authoritative: stale DeerFlow mappings are
# omitted so discovery never advertises a Wiki that its detail route cannot open.
rows = [row for row in rows if str(row.get("weknora_id") or "") in remote_by_id]
mapping_ids = [str(row["id"]) for row in rows]
index_states: dict[str, dict[str, Any]] = {}
index_store = getattr(request.app.state, "llmwiki_index_store", None)
if index_store is not None and mapping_ids:
try:
index_states = {str(state.get("knowledge_base_mapping_id") or ""): state for state in await index_store.list_index_status(mapping_ids) if isinstance(state, dict)}
except Exception:
logger.warning("Failed to read public Wiki local-index summaries", exc_info=True)
semaphore = asyncio.Semaphore(_PUBLIC_WIKI_SUMMARY_CONCURRENCY)
async def build_summary(row: dict[str, Any]) -> dict[str, Any]:
mapping_id = str(row["id"])
remote_id = str(row["weknora_id"])
remote = remote_by_id[remote_id]
async with semaphore:
try:
remote = {**remote, **await client.get_knowledge_base(remote_id)}
except WeKnoraError as exc:
if exc.status_code == 404:
return {}
logger.warning(
"Failed to load public Wiki detail metadata: mapping_id=%s",
mapping_id,
exc_info=True,
)
try:
pages, page_total, complete = await _all_public_wiki_page_views(client, remote_id)
wiki = _public_wiki_summary(
mapping_id,
pages,
total=page_total,
complete=complete,
)
except WeKnoraError as exc:
logger.warning(
"Failed to load public Wiki page summary: mapping_id=%s",
mapping_id,
exc_info=True,
)
wiki = (
_public_wiki_summary(mapping_id, [], total=0, complete=True)
if exc.status_code == 404
else _unavailable_public_wiki_summary(mapping_id)
)
return {
**_mapping_view(
row,
remote,
actor_user_id="__public_wiki__",
is_admin=False,
),
"wiki": wiki,
"local_index": _public_local_index_summary(
request,
row,
index_states.get(mapping_id),
),
}
knowledge_bases = [summary for summary in await asyncio.gather(*(build_summary(row) for row in rows)) if summary]
return {"knowledge_bases": knowledge_bases, "total": len(knowledge_bases)}
@@ -2596,13 +2831,7 @@ async def list_public_wiki_pages(
except WeKnoraError as exc:
_raise_upstream(exc)
raw_pages = result.get("pages") if isinstance(result, dict) else []
pages = [
view
for page_item in (raw_pages if isinstance(raw_pages, list) else [])
if isinstance(page_item, dict)
for view in [_wiki_page_view(page_item)]
if view is not None
]
pages = [view for page_item in (raw_pages if isinstance(raw_pages, list) else []) if isinstance(page_item, dict) for view in [_wiki_page_view(page_item)] if view is not None]
return {**result, "pages": pages}
@@ -2650,11 +2879,7 @@ def _public_wiki_index_item(value: Any) -> dict[str, Any] | None:
"slug": slug,
"title": str(value.get("title") or slug),
"summary": str(value.get("summary") or ""),
"category_path": (
[str(item) for item in value.get("category_path", []) if str(item).strip()]
if isinstance(value.get("category_path"), list)
else []
),
"category_path": ([str(item) for item in value.get("category_path", []) if str(item).strip()] if isinstance(value.get("category_path"), list) else []),
"wiki_path": str(value.get("wiki_path") or ""),
"depth": int(value.get("depth") or 0),
}
@@ -2680,20 +2905,12 @@ async def _all_public_wiki_index(client: WeKnoraClient, remote_id: str) -> dict[
version = max(version, int(payload.get("version") or 0))
raw_groups = payload.get("groups") if isinstance(payload.get("groups"), list) else []
group = next(
(
value
for value in raw_groups
if isinstance(value, dict) and value.get("type") == page_type
),
(value for value in raw_groups if isinstance(value, dict) and value.get("type") == page_type),
{},
)
total = max(total, int(group.get("total") or 0))
raw_items = group.get("items") if isinstance(group.get("items"), list) else []
items.extend(
item
for value in raw_items
if (item := _public_wiki_index_item(value)) is not None
)
items.extend(item for value in raw_items if (item := _public_wiki_index_item(value)) is not None)
next_cursor = str(group.get("next_cursor") or "")
if not next_cursor or next_cursor in seen_cursors:
break
@@ -3201,12 +3418,7 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
except WikiIndexError as exc:
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
mapping_by_id = {str(row["id"]): row for row in rows}
result["results"] = [
_sanitize_conversation_deposit_value(item)
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
else item
for item in result.get("results") or []
]
result["results"] = [_sanitize_conversation_deposit_value(item) if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {})) else item for item in result.get("results") or []]
return result