feat: improve assistant knowledge import and database compatibility
This commit is contained in:
1 parent
fbd8dec48f
commit
6b724bd00d
51 files changed
+4393
-1021
No files matched your search
@@ -1623,23 +1623,40 @@ save fallback.
|
||||
## Skill/assistant knowledge invariants
|
||||
|
||||
Skill distillation is implemented in `deerflow.skill_knowledge` and
|
||||
`deerflow.persistence.skill_knowledge`; the single native assistant base is in
|
||||
`deerflow.persistence.skill_knowledge`; native assistant bases are in
|
||||
`deerflow.assistant_knowledge` and
|
||||
`deerflow.persistence.assistant_knowledge`. Gateway composition, remote adapters,
|
||||
the lease dispatcher, watcher/scheduler, and APIs remain in `app.gateway` so the
|
||||
harness-to-app import firewall is preserved.
|
||||
the lease dispatcher, and APIs remain in `app.gateway` so the harness-to-app
|
||||
import firewall is preserved.
|
||||
|
||||
Never execute a skill or read/upload/summarize its code and script files.
|
||||
Knowledge fingerprints exclude those files and Markdown code fences. All
|
||||
resyncs must stage and validate the knowledge digest before activation; the old
|
||||
snapshot remains authoritative on failure. A binding may have only one active
|
||||
item (`active_dedupe_key` plus atomic lease claims). Detach/delete operates on
|
||||
source contributions, not canonical rows, and shared entities/relations are
|
||||
retired only when their final contribution is gone.
|
||||
item (`active_dedupe_key` plus atomic lease claims). Skill distillation targets
|
||||
only writable WeKnora Wiki knowledge bases in this product version; reject direct
|
||||
assistant-base targets, non-Wiki/FAQ/conversation/readonly targets, and direct
|
||||
Neo4j writes. Detach/delete operates on source contributions, not canonical rows,
|
||||
and shared entities/relations are retired only when their final contribution is
|
||||
gone.
|
||||
|
||||
There is exactly one global assistant base (`知识梳理`). Ordinary users have
|
||||
read/search access and administrators own all mutations. WeKnora direct graph
|
||||
writes require `llmwiki.weknora.graph.direct_write_enabled`, backend-only
|
||||
credentials, and the fixed parameterized templates in
|
||||
`deerflow.skill_knowledge.graph_writer`; never accept Cypher from an API, model,
|
||||
skill, or frontend.
|
||||
Assistant knowledge has one global full-library base (`知识梳理总库`) and any
|
||||
number of admin-created custom bases. Ordinary users have read/search access and
|
||||
administrators own all mutations. Assistant bases only accept WeKnora export
|
||||
packages or imports triggered from a WeKnora mapping. Do not add direct skill
|
||||
imports or arbitrary local-file imports to assistant knowledge. Importing into a
|
||||
custom assistant base must also mirror the package into the global base for
|
||||
cross-source entity alignment.
|
||||
|
||||
## openGauss / GaussDB persistence invariant
|
||||
|
||||
`database.backend=gauss` is an application-ORM backend, parallel to `mysql`.
|
||||
It normalizes `database.gauss_url` to `opengauss+asyncpg://` and requires the
|
||||
`gauss` optional dependency (`asyncpg` plus `opengauss-sqlalchemy`). Never route
|
||||
a Gauss URL through SQLAlchemy's generic PostgreSQL dialect: the wire protocol
|
||||
may connect, but dialect initialization and schema DDL can fail against the
|
||||
Gauss server/version behavior. Keep the LangGraph checkpointer on the separately
|
||||
configured SQLite backend; the PostgreSQL checkpointer schema is not part of the
|
||||
Gauss compatibility contract. Treat both `postgresql` and `opengauss` dialect
|
||||
names as timezone-aware PostgreSQL-family engines in portable types and
|
||||
dialect-specific session settings.
|
||||
@@ -13,8 +13,8 @@ FROM python:3.12-slim-bookworm AS builder
|
||||
ARG NODE_MAJOR=22
|
||||
ARG APT_MIRROR
|
||||
ARG UV_INDEX_URL
|
||||
# Optional extras to install (e.g. "postgres" for PostgreSQL support)
|
||||
# Usage: docker build --build-arg UV_EXTRAS=postgres ...
|
||||
# Optional extras to install (e.g. "postgres" or "gauss")
|
||||
# Usage: docker build --build-arg UV_EXTRAS=gauss ...
|
||||
ARG UV_EXTRAS
|
||||
|
||||
# Optionally override apt mirror for restricted networks (e.g. APT_MIRROR=mirrors.aliyun.com)
|
||||
|
||||
@@ -957,26 +957,54 @@ sandbox-directory recreation cannot discard a completed rewrite.
|
||||
## Skill knowledge distillation and assistant knowledge
|
||||
|
||||
The Gateway now exposes an administrator-only skill distillation pipeline at
|
||||
`/api/skill-knowledge`. A matrix job can send up to 200 skills to multiple
|
||||
WeKnora or global assistant-knowledge targets. The scanner never reads or
|
||||
uploads code/script files, strips Markdown code fences, blocks credential-like
|
||||
files and content, safely inspects ZIP members, and converts supported
|
||||
Office/PDF/spreadsheet files off the event loop. Jobs use database claims and
|
||||
renewable leases, stage snapshots before activation, retain the prior active
|
||||
version on failure, support per-item retry and entity/relation review, and keep
|
||||
per-binding contribution records for safe detach/delete.
|
||||
`/api/skill-knowledge`. Each request can send up to 200 skills to one writable
|
||||
WeKnora Wiki knowledge base; multi-base distillation is done by repeated
|
||||
operations so the local binding remains `skill_name + target_id`. The scanner
|
||||
never reads or uploads code/script files, strips Markdown code fences, blocks
|
||||
credential-like files and content, safely inspects ZIP members, and converts
|
||||
supported Office/PDF/spreadsheet files off the event loop. Jobs use database
|
||||
claims and renewable leases, stage snapshots before activation, retain the
|
||||
prior active version on failure, support per-item retry, and keep per-binding
|
||||
contribution records for safe detach/delete. Direct assistant-base targets,
|
||||
non-Wiki/FAQ/conversation/readonly targets, and direct Neo4j writes are rejected
|
||||
for this product version.
|
||||
|
||||
Bindings may opt into debounced file watching and cron synchronization through
|
||||
`/api/skill-knowledge/auto-sync-settings`. Only non-code knowledge digests can
|
||||
mark a binding stale. WeKnora Wiki, document, FAQ, conversation-archive and
|
||||
read-only sidecar targets have separate projections. Optional Neo4j direct
|
||||
writes are disabled by default under `llmwiki.weknora.graph`; when enabled they
|
||||
use only fixed parameterized templates and Neo4j's transactional HTTP endpoint.
|
||||
|
||||
`/api/assistant-knowledge` owns exactly one global DeerFlow-native base named
|
||||
`知识梳理`. Reads and search are available to authenticated users; initialization,
|
||||
local-file import, WeKnora export/import, source reimport/delete, and revision
|
||||
rollback are admin-only. Imports align canonical entities and relations while
|
||||
retaining source contributions, publish source/entity/relation Wiki revisions,
|
||||
and remove canonical knowledge only after its final contribution disappears.
|
||||
`/api/assistant-knowledge` manages DeerFlow-native assistant knowledge bases.
|
||||
There is one global full-library base named `知识梳理总库`, plus admin-created
|
||||
custom assistant bases. Reads and search are available to authenticated users;
|
||||
global initialization, custom-base creation, WeKnora export/import, source
|
||||
reimport, and source deletion are admin-only. Assistant bases only accept
|
||||
WeKnora export packages or imports triggered from a WeKnora mapping; direct
|
||||
skill import and arbitrary local-file import return 410. Importing into a custom
|
||||
assistant base also mirrors the same package into the global base, where
|
||||
canonical entities and relations are aligned across all imported sources.
|
||||
The database schema is introduced by Alembic revision `20260901_01`.
|
||||
|
||||
## openGauss / GaussDB application database
|
||||
|
||||
The application ORM can use a PG-compatible openGauss/GaussDB database through
|
||||
the official `opengauss-sqlalchemy` async dialect. Install the optional driver
|
||||
set before enabling it:
|
||||
|
||||
```bash
|
||||
uv sync --extra gauss
|
||||
```
|
||||
|
||||
Configure the business database independently from the LangGraph checkpointer:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
backend: gauss
|
||||
gauss_url: $GAUSS_DATABASE_URL
|
||||
|
||||
checkpointer:
|
||||
type: sqlite
|
||||
connection_string: .deer-flow/data/deerflow.db
|
||||
```
|
||||
|
||||
`GAUSS_DATABASE_URL` should normally be
|
||||
`opengauss+asyncpg://user:password@host:26000/deerflow`. Plain
|
||||
`opengauss://`, `gaussdb://`, and PostgreSQL-style URLs are accepted and
|
||||
normalized to the openGauss dialect. Do not configure the LangGraph
|
||||
PostgreSQL checkpointer against GaussDB: its migrations are PostgreSQL-specific.
|
||||
For a production image, build with `--build-arg UV_EXTRAS=gauss`.
|
||||
@@ -362,13 +362,21 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
|
||||
weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
|
||||
if local_index_config.enabled:
|
||||
embedding_config = local_index_config.embedding
|
||||
embedding_configured = bool(
|
||||
embedding_config.enabled
|
||||
and embedding_config.base_url
|
||||
and embedding_config.model.strip()
|
||||
)
|
||||
if embedding_configured:
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
|
||||
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(embedding_config)
|
||||
if local_index_config.enabled and app.state.llmwiki_embedding is not None:
|
||||
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
|
||||
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
|
||||
app.state.llmwiki_vector_cache = WikiVectorCache(
|
||||
app.state.llmwiki_index_store,
|
||||
max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
|
||||
|
||||
@@ -40,6 +40,7 @@ _MAX_SOURCE_CHARS = 1400
|
||||
@dataclass(frozen=True)
|
||||
class LlmWikiRagContext:
|
||||
knowledge_base_ids: list[str]
|
||||
assistant_knowledge_base_ids: list[str]
|
||||
source_count: int
|
||||
prompt: str
|
||||
sources: list[dict[str, Any]]
|
||||
@@ -50,6 +51,10 @@ def _store(request: Request) -> LlmWikiStore | None:
|
||||
return getattr(request.app.state, "llmwiki_store", None)
|
||||
|
||||
|
||||
def _assistant_store(request: Request) -> Any | None:
|
||||
return getattr(request.app.state, "assistant_knowledge_store", None)
|
||||
|
||||
|
||||
async def _actor(request: Request) -> tuple[str, bool]:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
@@ -76,6 +81,13 @@ def _body_selected_ids(body: Any) -> list[str]:
|
||||
return []
|
||||
|
||||
|
||||
def _body_selected_assistant_ids(body: Any) -> list[str]:
|
||||
for source in (getattr(body, "context", None), getattr(body, "metadata", None)):
|
||||
if isinstance(source, Mapping) and "assistant_knowledge_base_ids" in source:
|
||||
return _as_id_list(source.get("assistant_knowledge_base_ids"))
|
||||
return []
|
||||
|
||||
|
||||
def _assistant_agent_id(body: Any) -> str | None:
|
||||
assistant_id = validate_agent_id(getattr(body, "assistant_id", None))
|
||||
if assistant_id:
|
||||
@@ -106,8 +118,17 @@ def resolve_requested_llmwiki_knowledge_base_ids(body: Any) -> list[str]:
|
||||
return _body_selected_ids(body)
|
||||
|
||||
|
||||
def resolve_requested_assistant_knowledge_base_ids(body: Any) -> list[str]:
|
||||
"""Return assistant knowledge bases explicitly selected for the run."""
|
||||
|
||||
return _body_selected_assistant_ids(body)
|
||||
|
||||
|
||||
def llmwiki_selection_requested(body: Any) -> bool:
|
||||
return bool(resolve_requested_llmwiki_knowledge_base_ids(body))
|
||||
return bool(
|
||||
resolve_requested_llmwiki_knowledge_base_ids(body)
|
||||
or resolve_requested_assistant_knowledge_base_ids(body)
|
||||
)
|
||||
|
||||
|
||||
def _latest_human_query(graph_input: dict[str, Any]) -> str:
|
||||
@@ -191,18 +212,50 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
|
||||
}
|
||||
|
||||
|
||||
def _source_from_assistant_result(result: dict[str, Any], index: int) -> dict[str, Any]:
|
||||
title = str(result.get("title") or f"助手知识库片段 {index}")
|
||||
matched_section = result.get("matched_section") if isinstance(result.get("matched_section"), Mapping) else {}
|
||||
content = str(
|
||||
matched_section.get("content")
|
||||
or result.get("content_markdown")
|
||||
or result.get("content")
|
||||
or result.get("summary")
|
||||
or ""
|
||||
)
|
||||
snippet = str(result.get("summary") or content)
|
||||
raw = {
|
||||
**result,
|
||||
"source": "助手知识库",
|
||||
"url": "/page/workspace/knowledge/assistant",
|
||||
}
|
||||
return {
|
||||
"index": index,
|
||||
"id": result.get("id") or result.get("slug") or f"assistant:{index}",
|
||||
"title": title,
|
||||
"type": "assistant_knowledge",
|
||||
"snippet": _truncate(snippet, 360),
|
||||
"content": content,
|
||||
"source": "助手知识库",
|
||||
"score": float(result.get("score") or 0),
|
||||
"url": "/page/workspace/knowledge/assistant",
|
||||
"skillName": "assistant_knowledge_search",
|
||||
"mode": "citation",
|
||||
"raw": raw,
|
||||
}
|
||||
|
||||
|
||||
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
if not sources:
|
||||
return "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
return "当前用户选择了知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
|
||||
lines = [
|
||||
"当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||||
"当前用户选择了知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||||
"请优先依据这些资料回答;如果资料不足,请明确说明不足之处,不要编造。",
|
||||
"引用规则:正文关键结论必须使用 [1]、[2] 这样的编号引用下方资料;只能引用存在的编号;不要在正文末尾重复输出完整参考文献列表,前端会展示引用卡片。",
|
||||
"",
|
||||
f"用户问题:{query}",
|
||||
"",
|
||||
f'<llmwiki_wiki_reference_batch source_count="{len(sources)}">',
|
||||
f'<knowledge_reference_batch source_count="{len(sources)}">',
|
||||
]
|
||||
for source in sources:
|
||||
meta = " / ".join(
|
||||
@@ -213,12 +266,12 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
)
|
||||
if item
|
||||
)
|
||||
lines.append(f"[{source['index']}] {source.get('title') or 'LLMWiki 片段'}")
|
||||
lines.append(f"[{source['index']}] {source.get('title') or '知识库片段'}")
|
||||
if meta:
|
||||
lines.append(f"来源:{meta}")
|
||||
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
|
||||
lines.append("")
|
||||
lines.append("</llmwiki_wiki_reference_batch>")
|
||||
lines.append("</knowledge_reference_batch>")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
@@ -229,52 +282,99 @@ async def build_llmwiki_rag_context(
|
||||
graph_input: dict[str, Any],
|
||||
) -> LlmWikiRagContext:
|
||||
selected_ids = resolve_requested_llmwiki_knowledge_base_ids(body)
|
||||
if not selected_ids:
|
||||
return LlmWikiRagContext([], 0, "", [], "")
|
||||
|
||||
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
|
||||
local_config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
store = _store(request)
|
||||
if store is None:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
assistant_selected_ids = resolve_requested_assistant_knowledge_base_ids(body)
|
||||
if not selected_ids and not assistant_selected_ids:
|
||||
return LlmWikiRagContext([], [], 0, "", [], "")
|
||||
|
||||
query = _latest_human_query(graph_input)
|
||||
if not query:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], "")
|
||||
|
||||
user_id, is_admin = await _actor(request)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for mapping_id in selected_ids:
|
||||
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
|
||||
rows.append(row)
|
||||
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if search_service is None:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
0,
|
||||
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
try:
|
||||
payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
|
||||
except WikiIndexError as exc:
|
||||
logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
|
||||
return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。", [], query)
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
sources: list[dict[str, Any]] = []
|
||||
for result in payload.get("results") or []:
|
||||
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
sources.append(_source_from_result(result, mapping, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
return LlmWikiRagContext(selected_ids, len(sources), _build_prompt(query, sources), sources, query)
|
||||
|
||||
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
|
||||
local_config = request.app.state.config.llmwiki.local_wiki_index
|
||||
user_id, is_admin = await _actor(request)
|
||||
|
||||
if selected_ids:
|
||||
llmwiki_available = (local_config.enabled and local_config.internal_search_enabled) or runtime.weknora_enabled
|
||||
store = _store(request)
|
||||
if not llmwiki_available or store is None:
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], query)
|
||||
else:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for mapping_id in selected_ids:
|
||||
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
|
||||
rows.append(row)
|
||||
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if search_service is None:
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
0,
|
||||
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
try:
|
||||
payload = (
|
||||
await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
|
||||
if search_service is not None
|
||||
else {"results": []}
|
||||
)
|
||||
except WikiIndexError as exc:
|
||||
logger.warning(
|
||||
"Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s",
|
||||
sanitize_log_param(user_id),
|
||||
len(rows),
|
||||
exc.code,
|
||||
)
|
||||
if not assistant_selected_ids:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
0,
|
||||
f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
payload = {"results": []}
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
for result in payload.get("results") or []:
|
||||
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
sources.append(_source_from_result(result, mapping, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
|
||||
if assistant_selected_ids and len(sources) < _MAX_SOURCES:
|
||||
assistant_store = _assistant_store(request)
|
||||
if assistant_store is not None:
|
||||
rows = await assistant_store.search_pages(
|
||||
base_ids=assistant_selected_ids,
|
||||
query=query,
|
||||
limit=_MAX_SOURCES - len(sources),
|
||||
include_chunks=True,
|
||||
embedding_client=getattr(request.app.state, "llmwiki_embedding", None),
|
||||
)
|
||||
for row in rows:
|
||||
sources.append(_source_from_assistant_result(row, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
break
|
||||
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
assistant_selected_ids,
|
||||
len(sources),
|
||||
_build_prompt(query, sources),
|
||||
sources,
|
||||
query,
|
||||
)
|
||||
|
||||
|
||||
def attach_llmwiki_rag_to_graph_input(
|
||||
@@ -294,6 +394,7 @@ def attach_llmwiki_rag_to_graph_input(
|
||||
existing = str(extra.get("prompt_prefix") or "").strip()
|
||||
extra["prompt_prefix"] = f"{existing}\n\n{rag.prompt}".strip() if existing else rag.prompt
|
||||
extra["llmwiki_knowledge_base_ids"] = rag.knowledge_base_ids
|
||||
extra["assistant_knowledge_base_ids"] = rag.assistant_knowledge_base_ids
|
||||
extra["llmwiki_reference_query"] = rag.query
|
||||
if rag.sources:
|
||||
extra["llmwiki_reference_sources"] = rag.sources
|
||||
|
||||
@@ -461,7 +461,7 @@ async def _set_analytics_read_timeout(session) -> None:
|
||||
- MariaDB: ``max_statement_time`` (seconds)
|
||||
- MySQL <= 5.6: no per-statement timeout variable — skip it entirely
|
||||
(5.6 raises "Unknown system variable" otherwise)
|
||||
- PostgreSQL: ``statement_timeout`` (milliseconds)
|
||||
- PostgreSQL/openGauss: ``statement_timeout`` (milliseconds)
|
||||
"""
|
||||
conn = await session.connection()
|
||||
dialect = conn.dialect
|
||||
@@ -474,7 +474,7 @@ async def _set_analytics_read_timeout(session) -> None:
|
||||
await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
|
||||
# else: MySQL <= 5.6 has no statement-timeout variable — skip silently.
|
||||
await _try_set_session_value(session, f"SET SESSION innodb_lock_wait_timeout = {_LEADERBOARD_LOCK_WAIT_SECONDS}")
|
||||
elif name == "postgresql":
|
||||
elif name in {"postgresql", "opengauss"}:
|
||||
await _try_set_session_value(session, f"SET LOCAL statement_timeout = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
|
||||
|
||||
|
||||
|
||||
@@ -1,49 +1,50 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import mimetypes
|
||||
import logging
|
||||
import mimetypes
|
||||
import time
|
||||
import zipfile
|
||||
import zipfile
|
||||
from collections import OrderedDict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import quote
|
||||
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, PlainTextResponse, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.path_utils import aresolve_thread_virtual_path
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
from fastapi.responses import FileResponse, PlainTextResponse, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.path_utils import aresolve_thread_virtual_path
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["artifacts"])
|
||||
|
||||
ACTIVE_CONTENT_MIME_TYPES = {
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
"image/svg+xml",
|
||||
}
|
||||
|
||||
EDITABLE_TEXT_SUFFIXES = {
|
||||
".md",
|
||||
".markdown",
|
||||
".txt",
|
||||
".json",
|
||||
".yaml",
|
||||
".yml",
|
||||
".csv",
|
||||
".xml",
|
||||
".log",
|
||||
}
|
||||
|
||||
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["artifacts"])
|
||||
|
||||
ACTIVE_CONTENT_MIME_TYPES = {
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
"image/svg+xml",
|
||||
}
|
||||
|
||||
EDITABLE_TEXT_SUFFIXES = {
|
||||
".md",
|
||||
".markdown",
|
||||
".txt",
|
||||
".json",
|
||||
".yaml",
|
||||
".yml",
|
||||
".csv",
|
||||
".xml",
|
||||
".log",
|
||||
}
|
||||
|
||||
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
|
||||
|
||||
ArtifactLibraryKind = Literal["all", "document", "spreadsheet", "text_data", "archive", "other"]
|
||||
ArtifactLibrarySource = Literal["all", "general_qa", "agent"]
|
||||
|
||||
_MEDIA_MIME_PREFIXES = ("image/", "audio/", "video/")
|
||||
_MEDIA_SUFFIXES = {
|
||||
@@ -121,35 +122,37 @@ _ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 8
|
||||
_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE = 500
|
||||
_artifact_library_cache: OrderedDict[str, tuple[float, list[dict]]] = OrderedDict()
|
||||
|
||||
|
||||
def coerce_artifact_virtual_path(path: str) -> str:
|
||||
"""Normalize an artifacts URL path to a sandbox virtual path.
|
||||
|
||||
Streaming write previews may request a bare filename such as
|
||||
``未命名文件.md`` before the model has emitted a real
|
||||
``/mnt/user-data/outputs/...`` path. Mapping a single path segment into
|
||||
the thread outputs directory turns that into a normal 404 (file not written
|
||||
yet) instead of HTTP 400 ``Path must start with /mnt/user-data``.
|
||||
"""
|
||||
stripped = path.strip().lstrip("/").replace("\\", "/")
|
||||
if not stripped:
|
||||
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
|
||||
if stripped == _VIRTUAL_USER_DATA_PREFIX or stripped.startswith(_VIRTUAL_USER_DATA_PREFIX + "/"):
|
||||
return stripped
|
||||
if "/" in stripped or stripped in {".", ".."}:
|
||||
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
|
||||
return f"{_VIRTUAL_USER_DATA_PREFIX}/outputs/{stripped}"
|
||||
|
||||
|
||||
class ArtifactUpdateRequest(BaseModel):
|
||||
content: str
|
||||
|
||||
def coerce_artifact_virtual_path(path: str) -> str:
|
||||
"""Normalize an artifacts URL path to a sandbox virtual path.
|
||||
|
||||
Streaming write previews may request a bare filename such as
|
||||
``未命名文件.md`` before the model has emitted a real
|
||||
``/mnt/user-data/outputs/...`` path. Mapping a single path segment into
|
||||
the thread outputs directory turns that into a normal 404 (file not written
|
||||
yet) instead of HTTP 400 ``Path must start with /mnt/user-data``.
|
||||
"""
|
||||
stripped = path.strip().lstrip("/").replace("\\", "/")
|
||||
if not stripped:
|
||||
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
|
||||
if stripped == _VIRTUAL_USER_DATA_PREFIX or stripped.startswith(_VIRTUAL_USER_DATA_PREFIX + "/"):
|
||||
return stripped
|
||||
if "/" in stripped or stripped in {".", ".."}:
|
||||
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
|
||||
return f"{_VIRTUAL_USER_DATA_PREFIX}/outputs/{stripped}"
|
||||
|
||||
|
||||
class ArtifactUpdateRequest(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
class ArtifactLibraryItem(BaseModel):
|
||||
thread_id: str
|
||||
thread_title: str
|
||||
agent_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
thread_type: str | None = None
|
||||
source_type: Literal["general_qa", "agent"]
|
||||
path: str
|
||||
name: str
|
||||
size_bytes: int
|
||||
@@ -163,85 +166,85 @@ class ArtifactLibraryResponse(BaseModel):
|
||||
offset: int
|
||||
limit: int
|
||||
items: list[ArtifactLibraryItem]
|
||||
|
||||
|
||||
def _build_content_disposition(disposition_type: str, filename: str) -> str:
|
||||
"""Build an RFC 5987 encoded Content-Disposition header value."""
|
||||
return f"{disposition_type}; filename*=UTF-8''{quote(filename)}"
|
||||
|
||||
|
||||
def _build_attachment_headers(filename: str, extra_headers: dict[str, str] | None = None) -> dict[str, str]:
|
||||
headers = {"Content-Disposition": _build_content_disposition("attachment", filename)}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
return headers
|
||||
|
||||
|
||||
def is_text_file_by_content(path: Path, sample_size: int = 8192) -> bool:
|
||||
"""Check if file is text by examining content for null bytes."""
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
chunk = f.read(sample_size)
|
||||
# Text files shouldn't contain null bytes
|
||||
return b"\x00" not in chunk
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _extract_file_from_skill_archive(zip_path: Path, internal_path: str) -> bytes | None:
|
||||
"""Extract a file from a .skill ZIP archive.
|
||||
|
||||
Args:
|
||||
zip_path: Path to the .skill file (ZIP archive).
|
||||
internal_path: Path to the file inside the archive (e.g., "SKILL.md").
|
||||
|
||||
Returns:
|
||||
The file content as bytes, or None if not found.
|
||||
"""
|
||||
if not zipfile.is_zipfile(zip_path):
|
||||
return None
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path, "r") as zip_ref:
|
||||
# List all files in the archive
|
||||
namelist = zip_ref.namelist()
|
||||
|
||||
# Try direct path first
|
||||
if internal_path in namelist:
|
||||
return zip_ref.read(internal_path)
|
||||
|
||||
# Try with any top-level directory prefix (e.g., "skill-name/SKILL.md")
|
||||
for name in namelist:
|
||||
if name.endswith("/" + internal_path) or name == internal_path:
|
||||
return zip_ref.read(name)
|
||||
|
||||
# Not found
|
||||
return None
|
||||
except (zipfile.BadZipFile, KeyError):
|
||||
return None
|
||||
|
||||
|
||||
def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
|
||||
"""Walk an outputs directory and return artifact listing entries.
|
||||
|
||||
Each entry has ``path`` (virtual ``mnt/user-data/outputs/...``), ``name``,
|
||||
``size_bytes``, ``mime_type``, and ``modified_at`` (ISO 8601 UTC). Symlinks
|
||||
that escape the outputs root are dropped to keep the listing scoped to the
|
||||
thread. The result is sorted by ``modified_at`` descending (newest first).
|
||||
"""
|
||||
if not outputs_dir.exists() or not outputs_dir.is_dir():
|
||||
return []
|
||||
|
||||
outputs_root = outputs_dir.resolve()
|
||||
files: list[dict] = []
|
||||
for entry in outputs_root.rglob("*"):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
try:
|
||||
resolved = entry.resolve()
|
||||
resolved.relative_to(outputs_root)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
|
||||
|
||||
def _build_content_disposition(disposition_type: str, filename: str) -> str:
|
||||
"""Build an RFC 5987 encoded Content-Disposition header value."""
|
||||
return f"{disposition_type}; filename*=UTF-8''{quote(filename)}"
|
||||
|
||||
|
||||
def _build_attachment_headers(filename: str, extra_headers: dict[str, str] | None = None) -> dict[str, str]:
|
||||
headers = {"Content-Disposition": _build_content_disposition("attachment", filename)}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
return headers
|
||||
|
||||
|
||||
def is_text_file_by_content(path: Path, sample_size: int = 8192) -> bool:
|
||||
"""Check if file is text by examining content for null bytes."""
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
chunk = f.read(sample_size)
|
||||
# Text files shouldn't contain null bytes
|
||||
return b"\x00" not in chunk
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _extract_file_from_skill_archive(zip_path: Path, internal_path: str) -> bytes | None:
|
||||
"""Extract a file from a .skill ZIP archive.
|
||||
|
||||
Args:
|
||||
zip_path: Path to the .skill file (ZIP archive).
|
||||
internal_path: Path to the file inside the archive (e.g., "SKILL.md").
|
||||
|
||||
Returns:
|
||||
The file content as bytes, or None if not found.
|
||||
"""
|
||||
if not zipfile.is_zipfile(zip_path):
|
||||
return None
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path, "r") as zip_ref:
|
||||
# List all files in the archive
|
||||
namelist = zip_ref.namelist()
|
||||
|
||||
# Try direct path first
|
||||
if internal_path in namelist:
|
||||
return zip_ref.read(internal_path)
|
||||
|
||||
# Try with any top-level directory prefix (e.g., "skill-name/SKILL.md")
|
||||
for name in namelist:
|
||||
if name.endswith("/" + internal_path) or name == internal_path:
|
||||
return zip_ref.read(name)
|
||||
|
||||
# Not found
|
||||
return None
|
||||
except (zipfile.BadZipFile, KeyError):
|
||||
return None
|
||||
|
||||
|
||||
def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
|
||||
"""Walk an outputs directory and return artifact listing entries.
|
||||
|
||||
Each entry has ``path`` (virtual ``mnt/user-data/outputs/...``), ``name``,
|
||||
``size_bytes``, ``mime_type``, and ``modified_at`` (ISO 8601 UTC). Symlinks
|
||||
that escape the outputs root are dropped to keep the listing scoped to the
|
||||
thread. The result is sorted by ``modified_at`` descending (newest first).
|
||||
"""
|
||||
if not outputs_dir.exists() or not outputs_dir.is_dir():
|
||||
return []
|
||||
|
||||
outputs_root = outputs_dir.resolve()
|
||||
files: list[dict] = []
|
||||
for entry in outputs_root.rglob("*"):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
try:
|
||||
resolved = entry.resolve()
|
||||
resolved.relative_to(outputs_root)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
try:
|
||||
relative = resolved.relative_to(outputs_root).as_posix()
|
||||
mime_type, _ = mimetypes.guess_type(resolved.name)
|
||||
@@ -251,18 +254,18 @@ def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
|
||||
# is being scanned. Treat that transient entry as absent instead of
|
||||
# failing the complete thread/library listing.
|
||||
continue
|
||||
files.append(
|
||||
{
|
||||
"path": f"mnt/user-data/outputs/{relative}",
|
||||
"name": resolved.name,
|
||||
"size_bytes": stat.st_size,
|
||||
"mime_type": mime_type,
|
||||
files.append(
|
||||
{
|
||||
"path": f"mnt/user-data/outputs/{relative}",
|
||||
"name": resolved.name,
|
||||
"size_bytes": stat.st_size,
|
||||
"mime_type": mime_type,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(),
|
||||
}
|
||||
)
|
||||
|
||||
files.sort(key=lambda f: f["modified_at"], reverse=True)
|
||||
return files
|
||||
}
|
||||
)
|
||||
|
||||
files.sort(key=lambda f: f["modified_at"], reverse=True)
|
||||
return files
|
||||
|
||||
|
||||
def artifact_library_kind(name: str) -> str:
|
||||
@@ -335,7 +338,10 @@ async def _scan_artifact_library(thread_store) -> list[dict]:
|
||||
|
||||
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
|
||||
agent_id = metadata.get("agent_id")
|
||||
agent_name = metadata.get("agent_name")
|
||||
thread_type = metadata.get("thread_type")
|
||||
normalized_agent_id = agent_id.strip() if isinstance(agent_id, str) else ""
|
||||
normalized_agent_name = agent_name.strip() if isinstance(agent_name, str) else ""
|
||||
title = str(row.get("display_name") or "新对话")
|
||||
items: list[dict] = []
|
||||
for file in files:
|
||||
@@ -345,8 +351,10 @@ async def _scan_artifact_library(thread_store) -> list[dict]:
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"thread_title": title,
|
||||
"agent_id": agent_id if isinstance(agent_id, str) and agent_id else None,
|
||||
"agent_id": normalized_agent_id or None,
|
||||
"agent_name": normalized_agent_name or None,
|
||||
"thread_type": thread_type if isinstance(thread_type, str) and thread_type else None,
|
||||
"source_type": "agent" if normalized_agent_id else "general_qa",
|
||||
**file,
|
||||
"file_kind": artifact_library_kind(str(file.get("name") or "")),
|
||||
}
|
||||
@@ -393,6 +401,7 @@ async def list_artifact_library(
|
||||
request: Request,
|
||||
query: str | None = Query(default=None, max_length=200),
|
||||
file_kind: ArtifactLibraryKind = Query(default="all"),
|
||||
source_type: ArtifactLibrarySource = Query(default="all"),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
refresh: bool = Query(default=False),
|
||||
@@ -404,14 +413,35 @@ async def list_artifact_library(
|
||||
explicit ``present_files`` call. Uploads and image/audio/video outputs are
|
||||
intentionally outside this collection.
|
||||
"""
|
||||
from app.gateway.deps import get_thread_store
|
||||
from app.gateway.deps import get_agent_store, get_thread_store
|
||||
|
||||
items = await _cached_artifact_library(get_thread_store(request), refresh=refresh)
|
||||
agent_ids = list(dict.fromkeys(str(item.get("agent_id") or "") for item in items if item.get("agent_id")))
|
||||
agent_names: dict[str, str] = {}
|
||||
if agent_ids:
|
||||
try:
|
||||
agents = await get_agent_store(request).list_by_ids(
|
||||
agent_ids,
|
||||
str(get_effective_user_id() or "default"),
|
||||
)
|
||||
agent_names = {str(agent.get("id") or ""): str(agent.get("name") or "").strip() for agent in agents if agent.get("id") and agent.get("name")}
|
||||
except Exception:
|
||||
logger.warning("Failed to resolve artifact library agent names", exc_info=True)
|
||||
|
||||
items_with_sources = [
|
||||
{
|
||||
**item,
|
||||
"agent_name": (agent_names.get(str(item.get("agent_id") or "")) or item.get("agent_name") or item.get("agent_id")),
|
||||
}
|
||||
for item in items
|
||||
]
|
||||
normalized_query = (query or "").strip().casefold()
|
||||
filtered = [
|
||||
item
|
||||
for item in items
|
||||
if (file_kind == "all" or item["file_kind"] == file_kind) and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold())
|
||||
for item in items_with_sources
|
||||
if (file_kind == "all" or item["file_kind"] == file_kind)
|
||||
and (source_type == "all" or item["source_type"] == source_type)
|
||||
and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold() or normalized_query in str(item.get("agent_name") or "").casefold())
|
||||
]
|
||||
page = filtered[offset : offset + limit]
|
||||
return ArtifactLibraryResponse(
|
||||
@@ -420,176 +450,176 @@ async def list_artifact_library(
|
||||
limit=limit,
|
||||
items=[ArtifactLibraryItem.model_validate(item) for item in page],
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/threads/{thread_id}/artifacts",
|
||||
summary="List Artifact Files",
|
||||
description="List all files under the thread's outputs directory (/mnt/user-data/outputs/). Returns an empty list if the thread has no outputs.",
|
||||
)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def list_artifacts(thread_id: str, request: Request) -> dict:
|
||||
"""列出 thread 在 ``outputs/`` 下生成的产物文件。
|
||||
|
||||
圆桌 Step 3(``HighFidelityReport``)对每个**子席位** thread_id 调用本接口,
|
||||
展示各智能体写入沙箱 ``mnt/user-data/outputs/`` 的文件;协调智能体 thread
|
||||
通常无独立交付文件。
|
||||
|
||||
遍历路径::
|
||||
|
||||
.deer-flow/users/{user_id}/threads/{thread_id}/user-data/outputs/
|
||||
|
||||
返回 ``{ "thread_id", "files": [{ path, name, size_bytes, mime_type, modified_at }] }``,
|
||||
目录不存在时 ``files`` 为空数组(非 404)。
|
||||
"""
|
||||
outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
|
||||
return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/threads/{thread_id}/artifacts/{path:path}",
|
||||
summary="Get Artifact File",
|
||||
description="Retrieve an artifact file generated by the AI agent. Text and binary files can be viewed inline, while active web content is always downloaded.",
|
||||
)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_artifact(thread_id: str, path: str, request: Request, download: bool = False) -> Response:
|
||||
"""Get an artifact file by its path.
|
||||
|
||||
The endpoint automatically detects file types and returns appropriate content types.
|
||||
Use the `download` query parameter to force file download for non-active content.
|
||||
|
||||
Args:
|
||||
thread_id: The thread ID.
|
||||
path: The artifact path with virtual prefix (e.g., mnt/user-data/outputs/file.txt).
|
||||
request: FastAPI request object (automatically injected).
|
||||
|
||||
Returns:
|
||||
The file content as a FileResponse with appropriate content type:
|
||||
- Active content (HTML/XHTML/SVG): Served as download attachment
|
||||
- Text files: Plain text with proper MIME type
|
||||
- Binary files: Inline display with download option
|
||||
|
||||
Raises:
|
||||
HTTPException:
|
||||
- 400 if path is invalid or not a file
|
||||
- 403 if access denied (path traversal detected)
|
||||
- 404 if file not found
|
||||
|
||||
Query Parameters:
|
||||
download (bool): If true, forces attachment download for file types that are
|
||||
otherwise returned inline or as plain text. Active HTML/XHTML/SVG content
|
||||
is always downloaded regardless of this flag.
|
||||
|
||||
Example:
|
||||
- Get text file inline: `/api/threads/abc123/artifacts/mnt/user-data/outputs/notes.txt`
|
||||
- Download file: `/api/threads/abc123/artifacts/mnt/user-data/outputs/data.csv?download=true`
|
||||
- Active web content such as `.html`, `.xhtml`, and `.svg` artifacts is always downloaded
|
||||
"""
|
||||
path = coerce_artifact_virtual_path(path)
|
||||
# Check if this is a request for a file inside a .skill archive (e.g., xxx.skill/SKILL.md)
|
||||
if ".skill/" in path:
|
||||
# Split the path at ".skill/" to get the ZIP file path and internal path
|
||||
skill_marker = ".skill/"
|
||||
marker_pos = path.find(skill_marker)
|
||||
skill_file_path = path[: marker_pos + len(".skill")] # e.g., "mnt/user-data/outputs/my-skill.skill"
|
||||
internal_path = path[marker_pos + len(skill_marker) :] # e.g., "SKILL.md"
|
||||
|
||||
actual_skill_path = await aresolve_thread_virtual_path(thread_id, skill_file_path)
|
||||
|
||||
if not actual_skill_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Skill file not found: {skill_file_path}")
|
||||
|
||||
if not actual_skill_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {skill_file_path}")
|
||||
|
||||
# Extract the file from the .skill archive
|
||||
content = _extract_file_from_skill_archive(actual_skill_path, internal_path)
|
||||
if content is None:
|
||||
raise HTTPException(status_code=404, detail=f"File '{internal_path}' not found in skill archive")
|
||||
|
||||
# Determine MIME type based on the internal file
|
||||
mime_type, _ = mimetypes.guess_type(internal_path)
|
||||
# Add cache headers to avoid repeated ZIP extraction (cache for 5 minutes)
|
||||
cache_headers = {"Cache-Control": "private, max-age=300"}
|
||||
download_name = Path(internal_path).name or actual_skill_path.stem
|
||||
if download or mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=_build_attachment_headers(download_name, cache_headers))
|
||||
|
||||
if mime_type and mime_type.startswith("text/"):
|
||||
return PlainTextResponse(content=content.decode("utf-8"), media_type=mime_type, headers=cache_headers)
|
||||
|
||||
# Default to plain text for unknown types that look like text
|
||||
try:
|
||||
return PlainTextResponse(content=content.decode("utf-8"), media_type="text/plain", headers=cache_headers)
|
||||
except UnicodeDecodeError:
|
||||
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=cache_headers)
|
||||
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, path)
|
||||
|
||||
logger.info(f"Resolving artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
|
||||
|
||||
if not actual_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
|
||||
|
||||
if not actual_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(actual_path)
|
||||
|
||||
if download:
|
||||
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
|
||||
|
||||
# Always force download for active content types to prevent script execution
|
||||
# in the application origin when users open generated artifacts.
|
||||
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
|
||||
|
||||
if mime_type and mime_type.startswith("text/"):
|
||||
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
|
||||
|
||||
if is_text_file_by_content(actual_path):
|
||||
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
|
||||
|
||||
return Response(content=actual_path.read_bytes(), media_type=mime_type, headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)})
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/threads/{thread_id}/artifacts/{path:path}",
|
||||
summary="Update Artifact File",
|
||||
description="Update a text artifact file in the current thread workspace.",
|
||||
)
|
||||
@require_permission("threads", "write", owner_check=True)
|
||||
async def update_artifact(
|
||||
thread_id: str,
|
||||
path: str,
|
||||
request: Request,
|
||||
body: ArtifactUpdateRequest,
|
||||
) -> Response:
|
||||
"""Update a text artifact.
|
||||
|
||||
This endpoint is intentionally limited to normal text artifacts. Files inside
|
||||
.skill archives and active web content remain read-only from the artifact UI.
|
||||
"""
|
||||
if ".skill/" in path:
|
||||
raise HTTPException(status_code=400, detail="Skill archive contents are read-only.")
|
||||
|
||||
path = coerce_artifact_virtual_path(path)
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, path)
|
||||
logger.info(f"Updating artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
|
||||
|
||||
if not actual_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
|
||||
|
||||
if not actual_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(actual_path)
|
||||
suffix = actual_path.suffix.lower()
|
||||
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
raise HTTPException(status_code=400, detail="Active web artifacts are read-only.")
|
||||
|
||||
if suffix not in EDITABLE_TEXT_SUFFIXES and not (mime_type and mime_type.startswith("text/")):
|
||||
raise HTTPException(status_code=400, detail="Only text artifacts can be edited.")
|
||||
|
||||
actual_path.write_text(body.content, encoding="utf-8")
|
||||
return PlainTextResponse(content=body.content, media_type=mime_type or "text/plain")
|
||||
|
||||
|
||||
@router.get(
|
||||
"/threads/{thread_id}/artifacts",
|
||||
summary="List Artifact Files",
|
||||
description="List all files under the thread's outputs directory (/mnt/user-data/outputs/). Returns an empty list if the thread has no outputs.",
|
||||
)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def list_artifacts(thread_id: str, request: Request) -> dict:
|
||||
"""列出 thread 在 ``outputs/`` 下生成的产物文件。
|
||||
|
||||
圆桌 Step 3(``HighFidelityReport``)对每个**子席位** thread_id 调用本接口,
|
||||
展示各智能体写入沙箱 ``mnt/user-data/outputs/`` 的文件;协调智能体 thread
|
||||
通常无独立交付文件。
|
||||
|
||||
遍历路径::
|
||||
|
||||
.deer-flow/users/{user_id}/threads/{thread_id}/user-data/outputs/
|
||||
|
||||
返回 ``{ "thread_id", "files": [{ path, name, size_bytes, mime_type, modified_at }] }``,
|
||||
目录不存在时 ``files`` 为空数组(非 404)。
|
||||
"""
|
||||
outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
|
||||
return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/threads/{thread_id}/artifacts/{path:path}",
|
||||
summary="Get Artifact File",
|
||||
description="Retrieve an artifact file generated by the AI agent. Text and binary files can be viewed inline, while active web content is always downloaded.",
|
||||
)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_artifact(thread_id: str, path: str, request: Request, download: bool = False) -> Response:
|
||||
"""Get an artifact file by its path.
|
||||
|
||||
The endpoint automatically detects file types and returns appropriate content types.
|
||||
Use the `download` query parameter to force file download for non-active content.
|
||||
|
||||
Args:
|
||||
thread_id: The thread ID.
|
||||
path: The artifact path with virtual prefix (e.g., mnt/user-data/outputs/file.txt).
|
||||
request: FastAPI request object (automatically injected).
|
||||
|
||||
Returns:
|
||||
The file content as a FileResponse with appropriate content type:
|
||||
- Active content (HTML/XHTML/SVG): Served as download attachment
|
||||
- Text files: Plain text with proper MIME type
|
||||
- Binary files: Inline display with download option
|
||||
|
||||
Raises:
|
||||
HTTPException:
|
||||
- 400 if path is invalid or not a file
|
||||
- 403 if access denied (path traversal detected)
|
||||
- 404 if file not found
|
||||
|
||||
Query Parameters:
|
||||
download (bool): If true, forces attachment download for file types that are
|
||||
otherwise returned inline or as plain text. Active HTML/XHTML/SVG content
|
||||
is always downloaded regardless of this flag.
|
||||
|
||||
Example:
|
||||
- Get text file inline: `/api/threads/abc123/artifacts/mnt/user-data/outputs/notes.txt`
|
||||
- Download file: `/api/threads/abc123/artifacts/mnt/user-data/outputs/data.csv?download=true`
|
||||
- Active web content such as `.html`, `.xhtml`, and `.svg` artifacts is always downloaded
|
||||
"""
|
||||
path = coerce_artifact_virtual_path(path)
|
||||
# Check if this is a request for a file inside a .skill archive (e.g., xxx.skill/SKILL.md)
|
||||
if ".skill/" in path:
|
||||
# Split the path at ".skill/" to get the ZIP file path and internal path
|
||||
skill_marker = ".skill/"
|
||||
marker_pos = path.find(skill_marker)
|
||||
skill_file_path = path[: marker_pos + len(".skill")] # e.g., "mnt/user-data/outputs/my-skill.skill"
|
||||
internal_path = path[marker_pos + len(skill_marker) :] # e.g., "SKILL.md"
|
||||
|
||||
actual_skill_path = await aresolve_thread_virtual_path(thread_id, skill_file_path)
|
||||
|
||||
if not actual_skill_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Skill file not found: {skill_file_path}")
|
||||
|
||||
if not actual_skill_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {skill_file_path}")
|
||||
|
||||
# Extract the file from the .skill archive
|
||||
content = _extract_file_from_skill_archive(actual_skill_path, internal_path)
|
||||
if content is None:
|
||||
raise HTTPException(status_code=404, detail=f"File '{internal_path}' not found in skill archive")
|
||||
|
||||
# Determine MIME type based on the internal file
|
||||
mime_type, _ = mimetypes.guess_type(internal_path)
|
||||
# Add cache headers to avoid repeated ZIP extraction (cache for 5 minutes)
|
||||
cache_headers = {"Cache-Control": "private, max-age=300"}
|
||||
download_name = Path(internal_path).name or actual_skill_path.stem
|
||||
if download or mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=_build_attachment_headers(download_name, cache_headers))
|
||||
|
||||
if mime_type and mime_type.startswith("text/"):
|
||||
return PlainTextResponse(content=content.decode("utf-8"), media_type=mime_type, headers=cache_headers)
|
||||
|
||||
# Default to plain text for unknown types that look like text
|
||||
try:
|
||||
return PlainTextResponse(content=content.decode("utf-8"), media_type="text/plain", headers=cache_headers)
|
||||
except UnicodeDecodeError:
|
||||
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=cache_headers)
|
||||
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, path)
|
||||
|
||||
logger.info(f"Resolving artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
|
||||
|
||||
if not actual_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
|
||||
|
||||
if not actual_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(actual_path)
|
||||
|
||||
if download:
|
||||
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
|
||||
|
||||
# Always force download for active content types to prevent script execution
|
||||
# in the application origin when users open generated artifacts.
|
||||
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
|
||||
|
||||
if mime_type and mime_type.startswith("text/"):
|
||||
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
|
||||
|
||||
if is_text_file_by_content(actual_path):
|
||||
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
|
||||
|
||||
return Response(content=actual_path.read_bytes(), media_type=mime_type, headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)})
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/threads/{thread_id}/artifacts/{path:path}",
|
||||
summary="Update Artifact File",
|
||||
description="Update a text artifact file in the current thread workspace.",
|
||||
)
|
||||
@require_permission("threads", "write", owner_check=True)
|
||||
async def update_artifact(
|
||||
thread_id: str,
|
||||
path: str,
|
||||
request: Request,
|
||||
body: ArtifactUpdateRequest,
|
||||
) -> Response:
|
||||
"""Update a text artifact.
|
||||
|
||||
This endpoint is intentionally limited to normal text artifacts. Files inside
|
||||
.skill archives and active web content remain read-only from the artifact UI.
|
||||
"""
|
||||
if ".skill/" in path:
|
||||
raise HTTPException(status_code=400, detail="Skill archive contents are read-only.")
|
||||
|
||||
path = coerce_artifact_virtual_path(path)
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, path)
|
||||
logger.info(f"Updating artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
|
||||
|
||||
if not actual_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
|
||||
|
||||
if not actual_path.is_file():
|
||||
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(actual_path)
|
||||
suffix = actual_path.suffix.lower()
|
||||
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
|
||||
raise HTTPException(status_code=400, detail="Active web artifacts are read-only.")
|
||||
|
||||
if suffix not in EDITABLE_TEXT_SUFFIXES and not (mime_type and mime_type.startswith("text/")):
|
||||
raise HTTPException(status_code=400, detail="Only text artifacts can be edited.")
|
||||
|
||||
actual_path.write_text(body.content, encoding="utf-8")
|
||||
return PlainTextResponse(content=body.content, media_type=mime_type or "text/plain")
|
||||
File diff suppressed because it is too large.
Load diff
@@ -172,21 +172,15 @@ async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[
|
||||
targets: list[dict[str, Any]] = []
|
||||
for target in body.targets:
|
||||
data = target.model_dump()
|
||||
if target.target_type == "weknora":
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(target.target_id, actor or "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail=f"普通知识库不存在:{target.target_id}")
|
||||
data["target_name"] = target.target_name or str(mapping.get("name") or target.target_id)
|
||||
data["remote_business_key"] = target.remote_business_key or f"weknora:kb:{mapping['weknora_id']}"
|
||||
else:
|
||||
base = await request.app.state.assistant_knowledge_store.get_base()
|
||||
if base is None:
|
||||
base = await request.app.state.assistant_knowledge_store.initialize(created_by=actor)
|
||||
if target.target_id not in {base["id"], "global"}:
|
||||
raise HTTPException(status_code=404, detail="助手知识库仅允许选择全局知识梳理库")
|
||||
data["target_id"] = base["id"]
|
||||
data["target_name"] = base["name"]
|
||||
data["remote_business_key"] = f"assistant:base:{base['id']}"
|
||||
if target.target_type != "weknora":
|
||||
raise HTTPException(status_code=422, detail="不支持技能直接归纳到助手知识库,请先归纳到 WeKnora 普通知识库")
|
||||
if target.target_mode != "wiki":
|
||||
raise HTTPException(status_code=422, detail="技能归纳目标仅支持 WeKnora Wiki 知识库")
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(target.target_id, actor or "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail=f"普通知识库不存在:{target.target_id}")
|
||||
data["target_name"] = target.target_name or str(mapping.get("name") or target.target_id)
|
||||
data["remote_business_key"] = target.remote_business_key or f"weknora:kb:{mapping['weknora_id']}"
|
||||
targets.append(data)
|
||||
job = await _store(request).create_job(
|
||||
skill_names=body.skill_names,
|
||||
|
||||
@@ -405,6 +405,7 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
|
||||
"reasoning_effort",
|
||||
"rag_mode_enabled",
|
||||
"llmwiki_knowledge_base_ids",
|
||||
"assistant_knowledge_base_ids",
|
||||
"writing_mode",
|
||||
"writing_artifact_path",
|
||||
"canvas_artifact_title",
|
||||
@@ -1113,7 +1114,7 @@ async def start_run(
|
||||
if prefix_settings.enabled and prefix_settings.prompt_prefix.strip():
|
||||
graph_input = _prepend_prompt_prefix(graph_input, prefix_settings.prompt_prefix.strip())
|
||||
llmwiki_rag = (
|
||||
LlmWikiRagContext([], 0, "", [], "")
|
||||
LlmWikiRagContext([], [], 0, "", [], "")
|
||||
if fixed_answer is not None
|
||||
else await build_llmwiki_rag_context(
|
||||
request=request,
|
||||
@@ -1125,12 +1126,14 @@ async def start_run(
|
||||
graph_input = attach_llmwiki_rag_to_graph_input(graph_input, llmwiki_rag)
|
||||
effective_rag_mode = True
|
||||
run_metadata["llmwiki_knowledge_base_ids"] = llmwiki_rag.knowledge_base_ids
|
||||
run_metadata["assistant_knowledge_base_ids"] = llmwiki_rag.assistant_knowledge_base_ids
|
||||
run_metadata["llmwiki_rag_source_count"] = llmwiki_rag.source_count
|
||||
try:
|
||||
await run_ctx.thread_store.update_metadata(
|
||||
thread_id,
|
||||
{
|
||||
"llmwiki_knowledge_base_ids": llmwiki_rag.knowledge_base_ids,
|
||||
"assistant_knowledge_base_ids": llmwiki_rag.assistant_knowledge_base_ids,
|
||||
"rag_mode_enabled": True,
|
||||
"rag_mode": True,
|
||||
},
|
||||
@@ -1150,6 +1153,8 @@ async def start_run(
|
||||
body_context["rag_mode_enabled"] = True
|
||||
if "llmwiki_knowledge_base_ids" not in body_context and llmwiki_rag.knowledge_base_ids:
|
||||
body_context["llmwiki_knowledge_base_ids"] = llmwiki_rag.knowledge_base_ids
|
||||
if "assistant_knowledge_base_ids" not in body_context and llmwiki_rag.assistant_knowledge_base_ids:
|
||||
body_context["assistant_knowledge_base_ids"] = llmwiki_rag.assistant_knowledge_base_ids
|
||||
if not llmwiki_rag.knowledge_base_ids:
|
||||
existing_excluded = list(body_context.get("excluded_tools") or getattr(body, "excluded_tools", None) or [])
|
||||
if "llmwiki_search" not in existing_excluded:
|
||||
|
||||
@@ -429,7 +429,11 @@ auth_login:
|
||||
database:
|
||||
backend: sqlite
|
||||
sqlite_dir: .deer-flow/data
|
||||
pool_size: 20 # 基础连接池(postgres/mysql 生效;sqlite 忽略)
|
||||
# 高斯/OpenGauss 业务库示例(启用前先执行 uv sync --extra gauss):
|
||||
# backend: gauss
|
||||
# gauss_url: $GAUSS_DATABASE_URL # opengauss+asyncpg://user:pass@host:26000/deerflow
|
||||
# checkpointer 请继续使用下方 sqlite;不要把 PostgreSQL 专用 checkpointer 迁移直接用于高斯。
|
||||
pool_size: 20 # 基础连接池(postgres/gauss/mysql 生效;sqlite 忽略)
|
||||
max_overflow: 20 # 高峰期可临时超出 pool_size 的额外连接数,真实上限 = pool_size + max_overflow
|
||||
pool_timeout: 30 # 池满时等待空闲连接的秒数,超时报错
|
||||
pool_recycle: 1800 # 连接回收周期(秒);防火墙/pgbouncer/MySQL wait_timeout 静默断连兜底。多 worker 部署:每 worker 上限 = pool_size + max_overflow,总数 × worker 数 < DB max_connections
|
||||
@@ -564,9 +568,10 @@ llmwiki:
|
||||
password: ""
|
||||
database: "neo4j"
|
||||
timeout_seconds: 30
|
||||
# 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。
|
||||
# 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步,
|
||||
# 且服务启动时只启动定时器,不会立刻全量向量化。
|
||||
# 本地 Wiki 向量化:用于普通知识库“下载向量化数据包”和助手知识库向量检索。
|
||||
# 已有相同内容哈希 + 相同 embedding fingerprint 的 Wiki 向量会直接复用,不会重复编码。
|
||||
# 如需导出时自动补建向量,把 enabled 改为 true,并填写下面 embedding 服务地址。
|
||||
# auto_sync=true 才会后台自动增量同步;服务启动时只启动定时器,不会立刻全量向量化。
|
||||
local_wiki_index:
|
||||
enabled: false
|
||||
# 免登录 POST /api/knowledge/vector-search 返回的 Wiki 一键跳转链接前缀。
|
||||
@@ -574,7 +579,7 @@ llmwiki:
|
||||
auto_sync: false
|
||||
database_batch_size: 50
|
||||
embedding:
|
||||
model: bge-m3
|
||||
model: bge-embedding-m3
|
||||
base_url: ""
|
||||
api_key: ""
|
||||
|
||||
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""File-backed assistant knowledge import/export packages.
|
||||
|
||||
The assistant knowledge base is fed by WeKnora-exported data. Keep that
|
||||
boundary explicit: a WeKnora export is first materialized as a durable JSONL
|
||||
package, then imported asynchronously from that package. JSONL keeps large
|
||||
exports append-friendly and avoids treating the HTTP request body as the job
|
||||
state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
PACKAGE_VERSION = 1
|
||||
PACKAGE_RECORD_TYPES = {
|
||||
"source",
|
||||
"wiki_page",
|
||||
"document",
|
||||
"chunk",
|
||||
"vector",
|
||||
"entity",
|
||||
"relation",
|
||||
"graph",
|
||||
}
|
||||
|
||||
|
||||
def package_path(root: Path, job_id: str) -> Path:
|
||||
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})
|
||||
return root / f"{safe}.jsonl"
|
||||
|
||||
|
||||
def write_jsonl_record(handle: Any, record_type: str, payload: Mapping[str, Any]) -> None:
|
||||
handle.write(json.dumps({"record_type": record_type, "payload": dict(payload)}, ensure_ascii=False))
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def write_package(path: Path, records: Iterable[tuple[str, Mapping[str, Any]]]) -> dict[str, int]:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
counts: dict[str, int] = {key: 0 for key in PACKAGE_RECORD_TYPES}
|
||||
tmp_path = path.with_suffix(path.suffix + ".tmp")
|
||||
with tmp_path.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
write_jsonl_record(
|
||||
handle,
|
||||
"source",
|
||||
{
|
||||
"package_version": PACKAGE_VERSION,
|
||||
},
|
||||
)
|
||||
counts["source"] += 1
|
||||
for record_type, payload in records:
|
||||
if record_type not in PACKAGE_RECORD_TYPES:
|
||||
continue
|
||||
write_jsonl_record(handle, record_type, payload)
|
||||
counts[record_type] = counts.get(record_type, 0) + 1
|
||||
tmp_path.replace(path)
|
||||
return counts
|
||||
|
||||
|
||||
def read_package(path: Path) -> dict[str, Any]:
|
||||
package: dict[str, Any] = {
|
||||
"source": {},
|
||||
"wiki_pages": [],
|
||||
"documents": [],
|
||||
"chunks": [],
|
||||
"vectors": [],
|
||||
"entities": [],
|
||||
"relations": [],
|
||||
"graph": {"nodes": [], "edges": []},
|
||||
}
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
record = json.loads(line)
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
record_type = str(record.get("record_type") or "")
|
||||
payload = record.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
if record_type == "source":
|
||||
package["source"] = {**package.get("source", {}), **payload}
|
||||
elif record_type == "wiki_page":
|
||||
package["wiki_pages"].append(payload)
|
||||
elif record_type == "document":
|
||||
package["documents"].append(payload)
|
||||
elif record_type == "chunk":
|
||||
package["chunks"].append(payload)
|
||||
elif record_type == "vector":
|
||||
package["vectors"].append(payload)
|
||||
elif record_type == "entity":
|
||||
package["entities"].append(payload)
|
||||
elif record_type == "relation":
|
||||
package["relations"].append(payload)
|
||||
elif record_type == "graph":
|
||||
nodes = payload.get("nodes")
|
||||
edges = payload.get("edges")
|
||||
if isinstance(nodes, list):
|
||||
package["graph"]["nodes"].extend(item for item in nodes if isinstance(item, dict))
|
||||
if isinstance(edges, list):
|
||||
package["graph"]["edges"].extend(item for item in edges if isinstance(item, dict))
|
||||
for key, value in payload.items():
|
||||
if key not in {"nodes", "edges"}:
|
||||
package["graph"][key] = value
|
||||
return package
|
||||
+68
-21
@@ -11,10 +11,15 @@ blocking, making a unified file safe for both workloads. Writers
|
||||
that contend for the lock wait via the default 5-second sqlite3
|
||||
busy timeout rather than failing immediately.
|
||||
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
|
||||
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
|
||||
Gauss mode: application ORM data uses the official openGauss SQLAlchemy
|
||||
dialect over asyncpg. LangGraph's PostgreSQL checkpointer is intentionally not
|
||||
reused for GaussDB; configure the legacy ``checkpointer:`` section separately
|
||||
(normally sqlite).
|
||||
|
||||
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
|
||||
not have a built-in MySQL backend in this project, so deployments that choose
|
||||
MySQL for business tables should configure the legacy ``checkpointer:``
|
||||
section separately (usually sqlite) to preserve conversation checkpoints.
|
||||
@@ -42,24 +47,40 @@ from typing import Literal
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres", "gauss", "mysql"] = Field(
|
||||
default="memory",
|
||||
description=(
|
||||
"Storage backend for application data. 'memory' for development "
|
||||
"(no persistence across restarts), 'sqlite' for single-node "
|
||||
"deployment, 'postgres' for production multi-node deployment, "
|
||||
"'gauss' for openGauss/GaussDB business tables, and 'mysql' for "
|
||||
"MySQL-backed business tables."
|
||||
),
|
||||
)
|
||||
sqlite_dir: str = Field(
|
||||
default=".deer-flow/data",
|
||||
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
|
||||
)
|
||||
postgres_url: str = Field(
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
"Use $DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: postgresql://user:pass@host:5432/deerflow "
|
||||
"(the +asyncpg driver suffix is added automatically where needed)."
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
gauss_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"openGauss/GaussDB connection URL for the application ORM. "
|
||||
"Use $GAUSS_DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: opengauss+asyncpg://user:pass@host:26000/deerflow. "
|
||||
"Plain opengauss:// or PostgreSQL-style URLs are normalized to the "
|
||||
"official asynchronous openGauss SQLAlchemy dialect."
|
||||
),
|
||||
)
|
||||
mysql_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
@@ -75,26 +96,26 @@ class DatabaseConfig(BaseModel):
|
||||
)
|
||||
pool_size: int = Field(
|
||||
default=5,
|
||||
description="Connection pool size for the app ORM engine (postgres/mysql only).",
|
||||
description="Connection pool size for the app ORM engine (postgres/gauss/mysql only).",
|
||||
)
|
||||
max_overflow: int = Field(
|
||||
default=10,
|
||||
description=(
|
||||
"Extra connections allowed beyond pool_size under load (postgres/mysql only). "
|
||||
"Extra connections allowed beyond pool_size under load (postgres/gauss/mysql only). "
|
||||
"The real connection ceiling is pool_size + max_overflow. SQLAlchemy default is 10."
|
||||
),
|
||||
)
|
||||
pool_timeout: int = Field(
|
||||
default=30,
|
||||
description=(
|
||||
"Seconds a caller waits for a free pooled connection before erroring (postgres/mysql only). "
|
||||
"Seconds a caller waits for a free pooled connection before erroring (postgres/gauss/mysql only). "
|
||||
"SQLAlchemy default is 30."
|
||||
),
|
||||
)
|
||||
pool_recycle: int = Field(
|
||||
default=1800,
|
||||
description=(
|
||||
"Recycle pooled connections after this many seconds (postgres/mysql only). "
|
||||
"Recycle pooled connections after this many seconds (postgres/gauss/mysql only). "
|
||||
"Prevents stale-connection errors when a firewall, pgbouncer, or MySQL "
|
||||
"wait_timeout silently drops idle connections. pool_pre_ping catches most "
|
||||
"cases, but pool_recycle is the belt-and-suspenders safety net. "
|
||||
@@ -133,12 +154,38 @@ class DatabaseConfig(BaseModel):
|
||||
"""SQLAlchemy async URL for the application ORM engine."""
|
||||
if self.backend == "sqlite":
|
||||
return f"sqlite+aiosqlite:///{self.sqlite_path}"
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
if self.backend == "mysql":
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
if self.backend == "gauss":
|
||||
url = self.gauss_url.strip()
|
||||
if not url:
|
||||
raise ValueError("database.gauss_url is required for backend='gauss'")
|
||||
|
||||
# GaussDB speaks the PostgreSQL wire protocol, but its server
|
||||
# version string and DDL capabilities are not identical to
|
||||
# PostgreSQL. Always select the official openGauss dialect instead
|
||||
# of silently sending the URL through SQLAlchemy's generic PG
|
||||
# dialect (the latter commonly connects and then fails while
|
||||
# SQLAlchemy initializes/creates the schema).
|
||||
for prefix in (
|
||||
"opengauss://",
|
||||
"gaussdb://",
|
||||
"postgres://",
|
||||
"postgresql://",
|
||||
"postgresql+asyncpg://",
|
||||
):
|
||||
if url.startswith(prefix):
|
||||
return "opengauss+asyncpg://" + url[len(prefix) :]
|
||||
if url.startswith("opengauss+asyncpg://"):
|
||||
return url
|
||||
raise ValueError(
|
||||
"database.gauss_url must use opengauss://, "
|
||||
"opengauss+asyncpg://, or a PostgreSQL-style URL"
|
||||
)
|
||||
if self.backend == "mysql":
|
||||
url = self.mysql_url
|
||||
if not url:
|
||||
raise ValueError("database.mysql_url is required for backend='mysql'")
|
||||
|
||||
@@ -73,7 +73,7 @@ class WeKnoraDirectGraphConfig(BaseModel):
|
||||
class LocalWikiEmbeddingConfig(BaseModel):
|
||||
enabled: bool = True
|
||||
provider: str = "openai"
|
||||
model: str = "bge-m3"
|
||||
model: str = "bge-embedding-m3"
|
||||
base_url: str = ""
|
||||
api_key: str = Field(default="", repr=False)
|
||||
dimensions: int | None = Field(
|
||||
|
||||
+1
-1
@@ -494,7 +494,7 @@ class WeKnoraClient:
|
||||
await self._request(
|
||||
"GET",
|
||||
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/graph",
|
||||
params={"mode": "overview", "limit": max(1, min(limit, 500))},
|
||||
params={"mode": "overview", "limit": max(1, min(limit, 10000))},
|
||||
)
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
|
||||
+34
-4
@@ -1,7 +1,37 @@
|
||||
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
|
||||
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.integrations.weknora.local_index.retrieval import (
|
||||
WikiRetrievalService,
|
||||
is_fully_vectorized,
|
||||
)
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
|
||||
__all__ = [
|
||||
"WikiIndexError",
|
||||
"WikiRetrievalService",
|
||||
"WikiVectorSearchService",
|
||||
"WikiSyncService",
|
||||
"is_fully_vectorized",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name in {"WikiRetrievalService", "is_fully_vectorized"}:
|
||||
from deerflow.integrations.weknora.local_index import retrieval
|
||||
|
||||
return getattr(retrieval, name)
|
||||
if name in {"WikiIndexError", "WikiVectorSearchService"}:
|
||||
from deerflow.integrations.weknora.local_index import search
|
||||
|
||||
return getattr(search, name)
|
||||
if name == "WikiSyncService":
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
|
||||
return WikiSyncService
|
||||
raise AttributeError(name)
|
||||
+34
-3
@@ -1,10 +1,10 @@
|
||||
"""ORM model for the single global DeerFlow assistant knowledge base."""
|
||||
"""ORM model for DeerFlow-native assistant knowledge bases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import Float, Index, Integer, LargeBinary, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -19,7 +19,12 @@ class AssistantKnowledgeBaseRow(Base):
|
||||
__tablename__ = "assistant_knowledge_base"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
|
||||
status: Mapped[str] = mapped_column(String(24), default="active", unique=True)
|
||||
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
|
||||
# ``global`` is the full-library aggregation base. ``custom`` bases are
|
||||
# user-created assistant knowledge bases fed from WeKnora export packages.
|
||||
# Nullable keeps startup column reconciliation safe for existing databases.
|
||||
base_kind: Mapped[str | None] = mapped_column(String(24), nullable=True, index=True)
|
||||
status: Mapped[str] = mapped_column(String(64), default="active")
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
|
||||
@@ -91,6 +96,32 @@ class AssistantKnowledgeWikiRevisionRow(Base):
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
class AssistantKnowledgeVectorRow(Base):
|
||||
__tablename__ = "assistant_knowledge_vectors"
|
||||
__table_args__ = (
|
||||
Index("ix_assistant_knowledge_vector_base_fingerprint", "base_id", "embedding_fingerprint"),
|
||||
Index("ix_assistant_knowledge_vector_page", "page_id"),
|
||||
Index("ix_assistant_knowledge_vector_source", "source_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
base_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
page_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
source_vector_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
section_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
|
||||
|
||||
|
||||
class AssistantKnowledgeEntityRow(Base):
|
||||
__tablename__ = "assistant_knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
|
||||
|
||||
+807
-30
@@ -1,15 +1,21 @@
|
||||
"""Repository for the single global assistant knowledge base."""
|
||||
"""Repository for DeerFlow-native assistant knowledge bases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import difflib
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
from sqlalchemy import delete, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector, encode_vector, normalize_vector
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeBaseRow,
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
@@ -18,11 +24,24 @@ from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeRelationContributionRow,
|
||||
AssistantKnowledgeRelationRow,
|
||||
AssistantKnowledgeSourceRow,
|
||||
AssistantKnowledgeVectorRow,
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
)
|
||||
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
|
||||
|
||||
GLOBAL_BASE_NAME = "知识梳理总库"
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VECTOR_FIELD_NAMES = (
|
||||
"embedding",
|
||||
"vector",
|
||||
"embedding_vector",
|
||||
"content_vector",
|
||||
"dense_vector",
|
||||
"dense",
|
||||
)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
@@ -35,31 +54,246 @@ def _dict(row: Any) -> dict[str, Any]:
|
||||
data[key] = value.isoformat()
|
||||
elif key.endswith("_json"):
|
||||
data[key[:-5]] = data.pop(key) or ([] if "aliases" in key or "evidence" in key else {})
|
||||
if isinstance(row, AssistantKnowledgeBaseRow):
|
||||
raw_status = str(data.get("status") or "")
|
||||
data["status"] = "active" if raw_status.startswith("active") else raw_status
|
||||
data["base_kind"] = data.get("base_kind") or "custom"
|
||||
data["is_global"] = data["base_kind"] == "global"
|
||||
data["description"] = data.get("description") or ""
|
||||
return data
|
||||
|
||||
|
||||
def _stable_slug_part(value: Any) -> str:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
raw = str(uuid4())
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def _content_hash(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _first_vector_value(raw: dict[str, Any]) -> Any:
|
||||
for key in VECTOR_FIELD_NAMES:
|
||||
value = raw.get(key)
|
||||
if value is not None:
|
||||
return value
|
||||
vectors = raw.get("vectors")
|
||||
if isinstance(vectors, dict):
|
||||
for key in VECTOR_FIELD_NAMES:
|
||||
value = vectors.get(key)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _vector_blob_from_payload(raw: dict[str, Any]) -> tuple[bytes, int] | None:
|
||||
blob64 = raw.get("vector_blob_base64") or raw.get("embedding_blob_base64")
|
||||
dimensions = raw.get("embedding_dimensions") or raw.get("dimensions")
|
||||
if blob64 and dimensions:
|
||||
try:
|
||||
decoded = base64.b64decode(str(blob64), validate=True)
|
||||
vector = decode_vector(decoded, int(dimensions))
|
||||
return encode_vector(vector)
|
||||
except (ValueError, TypeError, binascii.Error):
|
||||
return None
|
||||
value = _first_vector_value(raw)
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, dict):
|
||||
value = _first_vector_value(value)
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return None
|
||||
try:
|
||||
return encode_vector(value, dimensions=int(dimensions) if dimensions else None)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
class AssistantKnowledgeRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def get_base(self) -> dict[str, Any] | None:
|
||||
stmt = select(AssistantKnowledgeBaseRow).where(AssistantKnowledgeBaseRow.status == "active").limit(1)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
async def _base_with_stats(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
row: AssistantKnowledgeBaseRow,
|
||||
) -> dict[str, Any]:
|
||||
data = _dict(row)
|
||||
data["source_count"] = int(
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count(AssistantKnowledgeSourceRow.id)).where(
|
||||
AssistantKnowledgeSourceRow.base_id == row.id,
|
||||
AssistantKnowledgeSourceRow.status != "deleted",
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
or 0
|
||||
)
|
||||
data["page_count"] = int(
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count(AssistantKnowledgeWikiPageRow.id)).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == row.id,
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
or 0
|
||||
)
|
||||
data["entity_count"] = int(
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count(AssistantKnowledgeEntityRow.id)).where(
|
||||
AssistantKnowledgeEntityRow.base_id == row.id,
|
||||
AssistantKnowledgeEntityRow.status == "active",
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
or 0
|
||||
)
|
||||
data["relation_count"] = int(
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count(AssistantKnowledgeRelationRow.id)).where(
|
||||
AssistantKnowledgeRelationRow.base_id == row.id,
|
||||
AssistantKnowledgeRelationRow.status == "active",
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
or 0
|
||||
)
|
||||
return data
|
||||
|
||||
async def initialize(self, *, created_by: str | None, name: str = "知识梳理") -> dict[str, Any]:
|
||||
async def get_base(self, base_id: str | None = None) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
if base_id:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeBaseRow).where(
|
||||
AssistantKnowledgeBaseRow.id == base_id,
|
||||
AssistantKnowledgeBaseRow.status != "deleted",
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return await self._base_with_stats(session, row) if row else None
|
||||
|
||||
row = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeBaseRow)
|
||||
.where(
|
||||
AssistantKnowledgeBaseRow.status != "deleted",
|
||||
AssistantKnowledgeBaseRow.base_kind == "global",
|
||||
)
|
||||
.order_by(AssistantKnowledgeBaseRow.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
# Backward compatibility: earlier builds had a single active row
|
||||
# and no ``base_kind``. Promote the first surviving base into the
|
||||
# global full-library aggregation base.
|
||||
row = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeBaseRow)
|
||||
.where(AssistantKnowledgeBaseRow.status != "deleted")
|
||||
.order_by(AssistantKnowledgeBaseRow.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.base_kind = "global"
|
||||
if not row.description:
|
||||
row.description = "默认全库沉淀库,汇聚所有 WeKnora 导出的助手知识数据。"
|
||||
row.updated_at = _now()
|
||||
await session.flush()
|
||||
await session.commit()
|
||||
return await self._base_with_stats(session, row) if row else None
|
||||
|
||||
async def list_bases(self) -> list[dict[str, Any]]:
|
||||
await self.get_base()
|
||||
async with self._sf() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeBaseRow)
|
||||
.where(AssistantKnowledgeBaseRow.status != "deleted")
|
||||
.order_by(
|
||||
AssistantKnowledgeBaseRow.base_kind.desc(),
|
||||
AssistantKnowledgeBaseRow.updated_at.desc(),
|
||||
)
|
||||
)
|
||||
).scalars()
|
||||
values = [await self._base_with_stats(session, row) for row in rows]
|
||||
global_values = [item for item in values if item.get("is_global")]
|
||||
custom_values = sorted(
|
||||
[item for item in values if not item.get("is_global")],
|
||||
key=lambda item: item.get("updated_at") or "",
|
||||
reverse=True,
|
||||
)
|
||||
return global_values + custom_values
|
||||
|
||||
async def initialize(self, *, created_by: str | None, name: str = GLOBAL_BASE_NAME) -> dict[str, Any]:
|
||||
existing = await self.get_base()
|
||||
if existing:
|
||||
return existing
|
||||
row = AssistantKnowledgeBaseRow(id=str(uuid4()), name=name.strip() or "知识梳理", status="active", created_by=created_by, created_at=_now(), updated_at=_now())
|
||||
row = AssistantKnowledgeBaseRow(
|
||||
id=str(uuid4()),
|
||||
name=name.strip() or GLOBAL_BASE_NAME,
|
||||
description="默认全库沉淀库,汇聚所有 WeKnora 导出的助手知识数据。",
|
||||
base_kind="global",
|
||||
status="active",
|
||||
created_by=created_by,
|
||||
created_at=_now(),
|
||||
updated_at=_now(),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
data = _dict(row)
|
||||
data = await self._base_with_stats(session, row)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def create_base(
|
||||
self,
|
||||
*,
|
||||
created_by: str | None,
|
||||
name: str,
|
||||
description: str = "",
|
||||
) -> dict[str, Any]:
|
||||
row_id = str(uuid4())
|
||||
row = AssistantKnowledgeBaseRow(
|
||||
id=row_id,
|
||||
name=name.strip(),
|
||||
description=description.strip(),
|
||||
base_kind="custom",
|
||||
# Some upgraded SQLite/MySQL databases may still carry the old
|
||||
# unique(status) constraint from the single-base implementation and
|
||||
# its original VARCHAR(24) length. Use a compact per-row active
|
||||
# marker internally and normalize it in API responses.
|
||||
status=f"active:{row_id[:17]}",
|
||||
created_by=created_by,
|
||||
created_at=_now(),
|
||||
updated_at=_now(),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
data = await self._base_with_stats(session, row)
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def _ensure_base(self, base_id: str | None, *, created_by: str | None) -> dict[str, Any]:
|
||||
if base_id == "global":
|
||||
return await self.initialize(created_by=created_by)
|
||||
if base_id:
|
||||
base = await self.get_base(base_id)
|
||||
if base is None:
|
||||
raise ValueError("助手知识库不存在")
|
||||
return base
|
||||
return await self.initialize(created_by=created_by)
|
||||
|
||||
async def _upsert_source(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
@@ -181,8 +415,56 @@ class AssistantKnowledgeRepository:
|
||||
skill_name: str | None = None,
|
||||
mapping_id: str | None = None,
|
||||
upload_batch_id: str | None = None,
|
||||
base_id: str | None = None,
|
||||
mirror_to_global: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
base = await self.initialize(created_by=created_by)
|
||||
base = await self._ensure_base(base_id, created_by=created_by)
|
||||
result = await self._import_package_into_base(
|
||||
base=base,
|
||||
package=package,
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
trigger=trigger,
|
||||
created_by=created_by,
|
||||
skill_name=skill_name,
|
||||
mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
)
|
||||
if mirror_to_global and not base.get("is_global"):
|
||||
global_base = await self.initialize(created_by=created_by)
|
||||
mirror = await self._import_package_into_base(
|
||||
base=global_base,
|
||||
package=package,
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
trigger=f"{trigger}:global_mirror"[:48],
|
||||
created_by=created_by,
|
||||
skill_name=skill_name,
|
||||
mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
)
|
||||
result["mirrored_global_base_id"] = global_base["id"]
|
||||
result["mirrored_global_job_id"] = mirror["id"]
|
||||
return result
|
||||
|
||||
async def create_queued_import_job(
|
||||
self,
|
||||
*,
|
||||
source_type: str,
|
||||
source_key: str,
|
||||
source_name: str,
|
||||
trigger: str,
|
||||
created_by: str | None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
skill_name: str | None = None,
|
||||
mapping_id: str | None = None,
|
||||
upload_batch_id: str | None = None,
|
||||
base_id: str | None = None,
|
||||
phase: str = "queued",
|
||||
) -> dict[str, Any]:
|
||||
base = await self._ensure_base(base_id, created_by=created_by)
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
source = await self._upsert_source(
|
||||
@@ -191,22 +473,18 @@ class AssistantKnowledgeRepository:
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
metadata=package.get("source") or {},
|
||||
metadata=metadata or {},
|
||||
skill_name=skill_name,
|
||||
mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
)
|
||||
# A re-import replaces this source's prior contributions while retaining
|
||||
# canonical entities/relations that are still supported by other sources.
|
||||
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
|
||||
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
|
||||
job = AssistantKnowledgeImportJobRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
source_id=source.id,
|
||||
trigger=trigger,
|
||||
status="running",
|
||||
phase="aligning_entities",
|
||||
status="queued",
|
||||
phase=phase,
|
||||
counts_json={},
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
@@ -214,6 +492,166 @@ class AssistantKnowledgeRepository:
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
source.last_import_job_id = job.id
|
||||
result = _dict(job)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
async def update_import_job(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
status: str | None = None,
|
||||
phase: str | None = None,
|
||||
counts: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
completed: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
job = await session.get(AssistantKnowledgeImportJobRow, job_id)
|
||||
if job is None:
|
||||
return None
|
||||
if status is not None:
|
||||
job.status = status
|
||||
if phase is not None:
|
||||
job.phase = phase
|
||||
if counts is not None:
|
||||
job.counts_json = counts
|
||||
if error is not None:
|
||||
job.last_error_message = error[:2048]
|
||||
elif status not in {None, "failed"}:
|
||||
job.last_error_message = None
|
||||
if completed:
|
||||
job.completed_at = _now()
|
||||
await session.flush()
|
||||
result = _dict(job)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
async def import_package_for_job(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
package: dict[str, Any],
|
||||
created_by: str | None,
|
||||
mirror_to_global: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
job = await session.get(AssistantKnowledgeImportJobRow, job_id)
|
||||
if job is None:
|
||||
raise ValueError("导入任务不存在")
|
||||
source = await session.get(AssistantKnowledgeSourceRow, job.source_id)
|
||||
base_row = await session.get(AssistantKnowledgeBaseRow, job.base_id)
|
||||
if source is None or base_row is None:
|
||||
raise ValueError("导入任务来源不存在")
|
||||
base = _dict(base_row)
|
||||
source_type = source.source_type
|
||||
source_key = source.source_key
|
||||
source_name = source.source_name
|
||||
source_id = source.id
|
||||
trigger = job.trigger
|
||||
skill_name = source.skill_name
|
||||
mapping_id = source.knowledge_base_mapping_id
|
||||
upload_batch_id = source.upload_batch_id
|
||||
|
||||
result = await self._import_package_into_base(
|
||||
base=base,
|
||||
package=package,
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
trigger=trigger,
|
||||
created_by=created_by,
|
||||
skill_name=skill_name,
|
||||
mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
existing_source_id=source_id,
|
||||
existing_job_id=job_id,
|
||||
)
|
||||
if mirror_to_global and not base.get("is_global"):
|
||||
global_base = await self.initialize(created_by=created_by)
|
||||
mirror = await self._import_package_into_base(
|
||||
base=global_base,
|
||||
package=package,
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
trigger=f"{trigger}:global_mirror"[:48],
|
||||
created_by=created_by,
|
||||
skill_name=skill_name,
|
||||
mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
)
|
||||
result["mirrored_global_base_id"] = global_base["id"]
|
||||
result["mirrored_global_job_id"] = mirror["id"]
|
||||
return result
|
||||
|
||||
async def _import_package_into_base(
|
||||
self,
|
||||
*,
|
||||
base: dict[str, Any],
|
||||
package: dict[str, Any],
|
||||
source_type: str,
|
||||
source_key: str,
|
||||
source_name: str,
|
||||
trigger: str,
|
||||
created_by: str | None,
|
||||
skill_name: str | None = None,
|
||||
mapping_id: str | None = None,
|
||||
upload_batch_id: str | None = None,
|
||||
existing_source_id: str | None = None,
|
||||
existing_job_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
if existing_source_id:
|
||||
source = await session.get(AssistantKnowledgeSourceRow, existing_source_id)
|
||||
if source is None or source.base_id != base["id"]:
|
||||
raise ValueError("导入任务来源不存在")
|
||||
source.source_name = source_name
|
||||
source.status = "active"
|
||||
source.metadata_json = package.get("source") or {}
|
||||
source.updated_at = now
|
||||
else:
|
||||
source = await self._upsert_source(
|
||||
session,
|
||||
base_id=base["id"],
|
||||
source_type=source_type,
|
||||
source_key=source_key,
|
||||
source_name=source_name,
|
||||
metadata=package.get("source") or {},
|
||||
skill_name=skill_name,
|
||||
mapping_id=mapping_id,
|
||||
upload_batch_id=upload_batch_id,
|
||||
)
|
||||
# A re-import replaces this source's prior contributions while retaining
|
||||
# canonical entities/relations that are still supported by other sources.
|
||||
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source.id))
|
||||
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source.id))
|
||||
await session.execute(delete(AssistantKnowledgeVectorRow).where(AssistantKnowledgeVectorRow.source_id == source.id))
|
||||
if existing_job_id:
|
||||
job = await session.get(AssistantKnowledgeImportJobRow, existing_job_id)
|
||||
if job is None or job.base_id != base["id"]:
|
||||
raise ValueError("导入任务不存在")
|
||||
job.source_id = source.id
|
||||
job.status = "running"
|
||||
job.phase = "aligning_entities"
|
||||
job.last_error_message = None
|
||||
job.completed_at = None
|
||||
else:
|
||||
job = AssistantKnowledgeImportJobRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
source_id=source.id,
|
||||
trigger=trigger,
|
||||
status="running",
|
||||
phase="aligning_entities",
|
||||
counts_json={},
|
||||
created_by=created_by,
|
||||
created_at=now,
|
||||
)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
source.last_import_job_id = job.id
|
||||
|
||||
entity_map: dict[str, str] = {}
|
||||
entity_rows: dict[str, tuple[AssistantKnowledgeEntityRow, dict[str, Any]]] = {}
|
||||
@@ -363,6 +801,7 @@ class AssistantKnowledgeRepository:
|
||||
wiki = package["wiki"]
|
||||
page = wiki.get("skill_page") if isinstance(wiki, dict) else None
|
||||
pages = [page] if page else []
|
||||
page_ids_by_slug: dict[str, str] = {}
|
||||
for raw in pages:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
@@ -404,16 +843,139 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
session.add(revision)
|
||||
page.title = title
|
||||
page.page_type = str(raw.get("page_type") or "source")
|
||||
page.content_markdown = content
|
||||
page.summary = summary
|
||||
page.search_text = "\n".join([title, summary, content, " ".join(raw.get("aliases") or [])])
|
||||
page.current_revision_id = revision.id
|
||||
page.status = "active"
|
||||
page.updated_at = now
|
||||
page_ids_by_slug[slug] = page.id
|
||||
|
||||
chunks = list(package.get("chunks") or [])
|
||||
chunk_pages = 0
|
||||
imported_vectors = 0
|
||||
for index, raw in enumerate(chunks):
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
content = str(
|
||||
raw.get("content")
|
||||
or raw.get("section_content")
|
||||
or raw.get("text")
|
||||
or raw.get("chunk_content")
|
||||
or ""
|
||||
).strip()
|
||||
if not content:
|
||||
continue
|
||||
raw_chunk_id = str(raw.get("id") or raw.get("chunk_id") or raw.get("remote_id") or f"{source.id}:{index}")
|
||||
document_id = str(raw.get("source_document_id") or raw.get("knowledge_id") or raw.get("document_id") or "")
|
||||
title = str(
|
||||
raw.get("title")
|
||||
or raw.get("knowledge_title")
|
||||
or raw.get("filename")
|
||||
or raw.get("knowledge_filename")
|
||||
or f"{source_name} 片段 {index + 1}"
|
||||
)
|
||||
heading = str(raw.get("heading") or raw.get("section_title") or "")
|
||||
summary = content[:240]
|
||||
page_content = "\n".join(
|
||||
item
|
||||
for item in [
|
||||
f"# {title}",
|
||||
"",
|
||||
f"- 来源:{source_name}",
|
||||
f"- 文档ID:{document_id}" if document_id else "",
|
||||
f"- 片段ID:{raw_chunk_id}" if raw_chunk_id else "",
|
||||
f"- 标题层级:{heading}" if heading else "",
|
||||
"",
|
||||
content,
|
||||
]
|
||||
if item != ""
|
||||
)
|
||||
chunk_page = await self._publish_page(
|
||||
session,
|
||||
base_id=base["id"],
|
||||
slug=f"chunks/{source.id}/{_stable_slug_part(raw_chunk_id)}",
|
||||
title=title,
|
||||
page_type="chunk",
|
||||
content=page_content,
|
||||
summary=summary,
|
||||
search_terms=[raw_chunk_id, document_id, heading],
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
created_by=created_by,
|
||||
change_type="chunk_import",
|
||||
)
|
||||
page_ids_by_slug[chunk_page.slug] = chunk_page.id
|
||||
vector_blob = _vector_blob_from_payload(raw)
|
||||
if vector_blob is not None:
|
||||
blob, dimensions = vector_blob
|
||||
session.add(
|
||||
AssistantKnowledgeVectorRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
page_id=chunk_page.id,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
source_vector_id=str(raw.get("vector_id") or raw.get("embedding_id") or raw_chunk_id),
|
||||
source_chunk_id=raw_chunk_id,
|
||||
section_index=int(raw.get("section_index") or raw.get("chunk_index") or index),
|
||||
heading=heading or None,
|
||||
section_content=content,
|
||||
content_hash=str(raw.get("content_hash") or _content_hash(content)),
|
||||
embedding_model=str(raw.get("embedding_model") or raw.get("model") or ""),
|
||||
embedding_fingerprint=str(raw.get("embedding_fingerprint") or ""),
|
||||
embedding_dimensions=dimensions,
|
||||
vector_blob=blob,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
imported_vectors += 1
|
||||
chunk_pages += 1
|
||||
|
||||
for index, raw in enumerate(list(package.get("vectors") or [])):
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "").strip("/")
|
||||
page_id = page_ids_by_slug.get(slug)
|
||||
if not page_id:
|
||||
continue
|
||||
content = str(raw.get("section_content") or raw.get("content") or raw.get("text") or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
vector_blob = _vector_blob_from_payload(raw)
|
||||
if vector_blob is None:
|
||||
continue
|
||||
blob, dimensions = vector_blob
|
||||
raw_chunk_id = str(raw.get("source_chunk_id") or raw.get("chunk_id") or "")
|
||||
session.add(
|
||||
AssistantKnowledgeVectorRow(
|
||||
id=str(uuid4()),
|
||||
base_id=base["id"],
|
||||
page_id=page_id,
|
||||
source_id=source.id,
|
||||
import_job_id=job.id,
|
||||
source_vector_id=str(raw.get("id") or raw.get("vector_id") or f"{slug}:{index}"),
|
||||
source_chunk_id=raw_chunk_id or None,
|
||||
section_index=int(raw.get("section_index") or index),
|
||||
heading=str(raw.get("heading") or "") or None,
|
||||
section_content=content,
|
||||
content_hash=str(raw.get("content_hash") or _content_hash(content)),
|
||||
embedding_model=str(raw.get("embedding_model") or raw.get("model") or ""),
|
||||
embedding_fingerprint=str(raw.get("embedding_fingerprint") or ""),
|
||||
embedding_dimensions=dimensions,
|
||||
vector_blob=blob,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
imported_vectors += 1
|
||||
|
||||
counts = {
|
||||
"wiki_pages": len(pages) + len(entity_rows) + len(relation_rows),
|
||||
"documents": len(package.get("documents") or []),
|
||||
"chunks": len(package.get("chunks") or []),
|
||||
"chunks": len(chunks),
|
||||
"chunk_pages": chunk_pages,
|
||||
"vectors": imported_vectors,
|
||||
"entities": len(raw_entities),
|
||||
"relations": len(raw_relations),
|
||||
"graph_nodes": len((package.get("graph") or {}).get("nodes") or []),
|
||||
@@ -439,8 +1001,24 @@ class AssistantKnowledgeRepository:
|
||||
contribution = (await session.execute(select(AssistantKnowledgeRelationContributionRow.id).where(AssistantKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
|
||||
relation.status = "active" if contribution else "deleted"
|
||||
|
||||
async def list_pages(self, *, query: str | None = None, page_type: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.status == "active")
|
||||
async def list_pages(
|
||||
self,
|
||||
*,
|
||||
base_id: str | None = None,
|
||||
query: str | None = None,
|
||||
page_type: str | None = None,
|
||||
limit: int = 100,
|
||||
include_chunks: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
if base is None:
|
||||
return []
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == base["id"],
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
)
|
||||
if not include_chunks and not page_type:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
|
||||
if query:
|
||||
needle = f"%{query.strip()}%"
|
||||
stmt = stmt.where(or_(AssistantKnowledgeWikiPageRow.title.ilike(needle), AssistantKnowledgeWikiPageRow.search_text.ilike(needle)))
|
||||
@@ -450,15 +1028,178 @@ class AssistantKnowledgeRepository:
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_page(self, slug: str) -> dict[str, Any] | None:
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/"), AssistantKnowledgeWikiPageRow.status == "active")
|
||||
async def search_pages(
|
||||
self,
|
||||
*,
|
||||
base_ids: list[str],
|
||||
query: str,
|
||||
limit: int = 8,
|
||||
include_chunks: bool = True,
|
||||
embedding_client: Any | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved_base_ids: list[str] = []
|
||||
for base_id in list(dict.fromkeys(base_ids)):
|
||||
base = await self.get_base(base_id)
|
||||
if base is not None:
|
||||
resolved_base_ids.append(str(base["id"]))
|
||||
if not resolved_base_ids:
|
||||
return []
|
||||
if embedding_client is not None and query.strip():
|
||||
try:
|
||||
vector_results = await self._search_pages_by_vector(
|
||||
base_ids=resolved_base_ids,
|
||||
query=query,
|
||||
limit=limit,
|
||||
include_chunks=include_chunks,
|
||||
embedding_client=embedding_client,
|
||||
)
|
||||
if vector_results:
|
||||
return vector_results
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("Assistant knowledge vector search failed; falling back to keyword search", exc_info=True)
|
||||
tokens = [token for token in query.strip().lower().split() if token]
|
||||
needle = f"%{query.strip()}%" if query.strip() else "%"
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id.in_(resolved_base_ids),
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
)
|
||||
if not include_chunks:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
|
||||
if query.strip():
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
AssistantKnowledgeWikiPageRow.title.ilike(needle),
|
||||
AssistantKnowledgeWikiPageRow.search_text.ilike(needle),
|
||||
*[
|
||||
AssistantKnowledgeWikiPageRow.search_text.ilike(f"%{token}%")
|
||||
for token in tokens[:8]
|
||||
],
|
||||
)
|
||||
)
|
||||
stmt = stmt.order_by(AssistantKnowledgeWikiPageRow.updated_at.desc()).limit(max(limit * 8, limit))
|
||||
async with self._sf() as session:
|
||||
rows = [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
def score(row: dict[str, Any]) -> float:
|
||||
text = f"{row.get('title') or ''}\n{row.get('summary') or ''}\n{row.get('search_text') or row.get('content_markdown') or ''}".lower()
|
||||
if not tokens:
|
||||
return 0.0
|
||||
value = 0.0
|
||||
title = str(row.get("title") or "").lower()
|
||||
for token in tokens:
|
||||
if token in title:
|
||||
value += 3.0
|
||||
value += min(text.count(token), 8) * 0.25
|
||||
if row.get("page_type") == "chunk":
|
||||
value += 0.5
|
||||
return value
|
||||
|
||||
rows.sort(key=score, reverse=True)
|
||||
results = rows[:limit]
|
||||
for row in results:
|
||||
row["score"] = score(row)
|
||||
return results
|
||||
|
||||
async def _search_pages_by_vector(
|
||||
self,
|
||||
*,
|
||||
base_ids: list[str],
|
||||
query: str,
|
||||
limit: int,
|
||||
include_chunks: bool,
|
||||
embedding_client: Any,
|
||||
) -> list[dict[str, Any]]:
|
||||
raw_query_vector = await embedding_client.embed_query(query)
|
||||
configured_dimensions = getattr(embedding_client, "dimensions", None)
|
||||
query_vector = normalize_vector(raw_query_vector, dimensions=configured_dimensions)
|
||||
dimensions = int(query_vector.size)
|
||||
stmt = (
|
||||
select(AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow)
|
||||
.join(
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiPageRow.id == AssistantKnowledgeVectorRow.page_id,
|
||||
)
|
||||
.where(
|
||||
AssistantKnowledgeVectorRow.base_id.in_(base_ids),
|
||||
AssistantKnowledgeVectorRow.embedding_dimensions == dimensions,
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
)
|
||||
.order_by(AssistantKnowledgeVectorRow.created_at.desc())
|
||||
)
|
||||
if not include_chunks:
|
||||
stmt = stmt.where(AssistantKnowledgeWikiPageRow.page_type != "chunk")
|
||||
async with self._sf() as session:
|
||||
rows = list((await session.execute(stmt)).all())
|
||||
if not rows:
|
||||
return []
|
||||
tokens = [token for token in query.strip().lower().split() if token]
|
||||
ranked: list[tuple[float, AssistantKnowledgeVectorRow, AssistantKnowledgeWikiPageRow]] = []
|
||||
for vector, page in rows:
|
||||
try:
|
||||
stored = decode_vector(vector.vector_blob, int(vector.embedding_dimensions))
|
||||
except ValueError:
|
||||
continue
|
||||
score = float(np.dot(query_vector, stored))
|
||||
if tokens:
|
||||
haystack = f"{page.title}\n{page.summary}\n{vector.heading or ''}\n{vector.section_content}".lower()
|
||||
score += min(sum(1 for token in tokens if token in haystack), 4) * 0.02
|
||||
ranked.append((score, vector, page))
|
||||
ranked.sort(key=lambda item: item[0], reverse=True)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
seen_pages: set[str] = set()
|
||||
for score, vector, page in ranked:
|
||||
if len(results) >= limit:
|
||||
break
|
||||
page_data = _dict(page)
|
||||
page_data["score"] = score
|
||||
page_data["content_markdown"] = vector.section_content or page_data.get("content_markdown") or ""
|
||||
page_data["summary"] = page_data.get("summary") or (vector.section_content[:240] if vector.section_content else "")
|
||||
page_data["retrieval_mode"] = "vector"
|
||||
page_data["matched_section"] = {
|
||||
"id": vector.id,
|
||||
"section_index": int(vector.section_index),
|
||||
"heading": vector.heading,
|
||||
"content": vector.section_content,
|
||||
"score": score,
|
||||
"source_chunk_id": vector.source_chunk_id,
|
||||
"embedding_model": vector.embedding_model,
|
||||
"embedding_fingerprint": vector.embedding_fingerprint,
|
||||
}
|
||||
dedupe_key = f"{page.id}:{vector.section_index}:{vector.content_hash}"
|
||||
if page.id in seen_pages and page.page_type != "chunk":
|
||||
continue
|
||||
seen_pages.add(page.id)
|
||||
seen_pages.add(dedupe_key)
|
||||
results.append(page_data)
|
||||
return results
|
||||
|
||||
async def get_page(self, slug: str, *, base_id: str | None = None) -> dict[str, Any] | None:
|
||||
base = await self.get_base(base_id)
|
||||
if base is None:
|
||||
return None
|
||||
stmt = select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == base["id"],
|
||||
AssistantKnowledgeWikiPageRow.slug == slug.strip("/"),
|
||||
AssistantKnowledgeWikiPageRow.status == "active",
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def list_revisions(self, slug: str) -> list[dict[str, Any]]:
|
||||
async def list_revisions(self, slug: str, *, base_id: str | None = None) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
if base is None:
|
||||
return []
|
||||
async with self._sf() as session:
|
||||
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
|
||||
page = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == base["id"],
|
||||
AssistantKnowledgeWikiPageRow.slug == slug.strip("/"),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if page is None:
|
||||
return []
|
||||
stmt = select(AssistantKnowledgeWikiRevisionRow).where(AssistantKnowledgeWikiRevisionRow.page_id == page.id).order_by(AssistantKnowledgeWikiRevisionRow.revision_no.desc())
|
||||
@@ -484,10 +1225,27 @@ class AssistantKnowledgeRepository:
|
||||
)
|
||||
return {"left": left, "right": right, "diff": diff}
|
||||
|
||||
async def rollback(self, *, slug: str, revision_id: str, actor: str | None) -> dict[str, Any] | None:
|
||||
async def rollback(
|
||||
self,
|
||||
*,
|
||||
slug: str,
|
||||
revision_id: str,
|
||||
actor: str | None,
|
||||
base_id: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
base = await self.get_base(base_id)
|
||||
if base is None:
|
||||
return None
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
page = (await session.execute(select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.slug == slug.strip("/")))).scalar_one_or_none()
|
||||
page = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiPageRow).where(
|
||||
AssistantKnowledgeWikiPageRow.base_id == base["id"],
|
||||
AssistantKnowledgeWikiPageRow.slug == slug.strip("/"),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
source = await session.get(AssistantKnowledgeWikiRevisionRow, revision_id)
|
||||
if page is None or source is None or source.page_id != page.id:
|
||||
return None
|
||||
@@ -517,9 +1275,19 @@ class AssistantKnowledgeRepository:
|
||||
await session.commit()
|
||||
return data
|
||||
|
||||
async def list_sources(self) -> list[dict[str, Any]]:
|
||||
async def list_sources(self, *, base_id: str | None = None) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
if base is None:
|
||||
return []
|
||||
async with self._sf() as session:
|
||||
stmt = select(AssistantKnowledgeSourceRow).where(AssistantKnowledgeSourceRow.status != "deleted").order_by(AssistantKnowledgeSourceRow.updated_at.desc())
|
||||
stmt = (
|
||||
select(AssistantKnowledgeSourceRow)
|
||||
.where(
|
||||
AssistantKnowledgeSourceRow.base_id == base["id"],
|
||||
AssistantKnowledgeSourceRow.status != "deleted",
|
||||
)
|
||||
.order_by(AssistantKnowledgeSourceRow.updated_at.desc())
|
||||
)
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
@@ -547,9 +1315,17 @@ class AssistantKnowledgeRepository:
|
||||
result["contributions"] = [_dict(value) for value in contributions]
|
||||
return result
|
||||
|
||||
async def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
|
||||
async def list_jobs(self, limit: int = 100, *, base_id: str | None = None) -> list[dict[str, Any]]:
|
||||
base = await self.get_base(base_id)
|
||||
if base is None:
|
||||
return []
|
||||
async with self._sf() as session:
|
||||
stmt = select(AssistantKnowledgeImportJobRow).order_by(AssistantKnowledgeImportJobRow.created_at.desc()).limit(limit)
|
||||
stmt = (
|
||||
select(AssistantKnowledgeImportJobRow)
|
||||
.where(AssistantKnowledgeImportJobRow.base_id == base["id"])
|
||||
.order_by(AssistantKnowledgeImportJobRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
@@ -565,6 +1341,7 @@ class AssistantKnowledgeRepository:
|
||||
source.status = "deleted"
|
||||
await session.execute(delete(AssistantKnowledgeEntityContributionRow).where(AssistantKnowledgeEntityContributionRow.source_id == source_id))
|
||||
await session.execute(delete(AssistantKnowledgeRelationContributionRow).where(AssistantKnowledgeRelationContributionRow.source_id == source_id))
|
||||
await session.execute(delete(AssistantKnowledgeVectorRow).where(AssistantKnowledgeVectorRow.source_id == source_id))
|
||||
pages = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeWikiPageRow).where(AssistantKnowledgeWikiPageRow.current_revision_id.in_(select(AssistantKnowledgeWikiRevisionRow.id).where(AssistantKnowledgeWikiRevisionRow.source_id == source_id)))
|
||||
|
||||
@@ -162,7 +162,7 @@ async def _auto_create_postgres_db(url: str) -> None:
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
|
||||
logger.info("Auto-created PostgreSQL database: %s", db_name)
|
||||
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
@@ -216,10 +216,10 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
added column are created as well. Table-level ``UniqueConstraint`` objects
|
||||
that are declared on the model but missing from the table are also created
|
||||
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL alike) — without this a newly
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
|
||||
on an existing database, weakening dedupe.
|
||||
Works on sqlite/mysql/postgres — the column DDL is compiled through the
|
||||
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
|
||||
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
|
||||
``BeijingDateTime``) resolve to the right backend type. Everything is
|
||||
per-column best-effort: a failure is logged and startup continues.
|
||||
@@ -341,13 +341,13 @@ async def init_engine(
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", "postgres", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
|
||||
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Base connection pool size (postgres/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/mysql). Prevents stale
|
||||
pool_size: Base connection pool size (postgres/gauss/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
|
||||
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
"""
|
||||
@@ -365,11 +365,21 @@ async def init_engine(
|
||||
except RuntimeError:
|
||||
_engine_loop = None
|
||||
|
||||
if backend == "postgres":
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
|
||||
if backend in {"postgres", "gauss"}:
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
extra = "gauss" if backend == "gauss" else "postgres"
|
||||
raise ImportError(
|
||||
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
|
||||
f"Install it with:\n uv sync --extra {extra}\n"
|
||||
"Or switch to backend: sqlite in config.yaml for single-node deployment."
|
||||
) from None
|
||||
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
@@ -409,7 +419,7 @@ async def init_engine(
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend == "postgres":
|
||||
elif backend in {"postgres", "gauss"}:
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
@@ -454,7 +464,7 @@ async def init_engine(
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend == "postgres" and "does not exist" in str(exc):
|
||||
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database
|
||||
@@ -497,7 +507,7 @@ async def init_engine(
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
logger.info("Persistence engine initialized: backend=%s", backend)
|
||||
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
|
||||
|
||||
|
||||
async def init_engine_from_config(config) -> None:
|
||||
|
||||
+18
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -35,6 +36,23 @@ class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
|
||||
|
||||
async def iter_vector_snapshot_rows(
|
||||
self,
|
||||
mapping_id: str,
|
||||
fingerprint: str,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Iterate vector rows for streaming export.
|
||||
|
||||
Implementations with a database backend should override this method so
|
||||
very large vector packages do not need to be materialized in memory.
|
||||
"""
|
||||
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
|
||||
for row in snapshot.get("rows") or []:
|
||||
if isinstance(row, dict):
|
||||
yield row
|
||||
|
||||
@abstractmethod
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
|
||||
|
||||
|
||||
+36
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
@@ -181,6 +182,41 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
|
||||
|
||||
async def iter_vector_snapshot_rows(
|
||||
self,
|
||||
mapping_id: str,
|
||||
fingerprint: str,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
batch = max(1, min(int(batch_size), 5000))
|
||||
offset = 0
|
||||
while True:
|
||||
stmt = (
|
||||
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
|
||||
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
|
||||
.where(
|
||||
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
|
||||
LlmWikiWikiPageRow.index_status == "ready",
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
|
||||
.limit(batch)
|
||||
.offset(offset)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
pairs = list((await session.execute(stmt)).all())
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
yield row
|
||||
if len(rows) < batch:
|
||||
break
|
||||
offset += len(rows)
|
||||
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""Add assistant knowledge vector table.
|
||||
|
||||
Revision ID: 20260902_01
|
||||
Revises: 20260901_01
|
||||
Create Date: 2026-09-02
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.assistant_knowledge.model import AssistantKnowledgeVectorRow
|
||||
|
||||
revision = "20260902_01"
|
||||
down_revision = "20260901_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
AssistantKnowledgeVectorRow.__table__.create(op.get_bind(), checkfirst=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
AssistantKnowledgeVectorRow.__table__.drop(op.get_bind(), checkfirst=True)
|
||||
+1
@@ -43,6 +43,7 @@ from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeRelationContributionRow,
|
||||
AssistantKnowledgeRelationRow,
|
||||
AssistantKnowledgeSourceRow,
|
||||
AssistantKnowledgeVectorRow,
|
||||
AssistantKnowledgeWikiPageRow,
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import UTC, datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, DateTime, Text
|
||||
@@ -14,6 +14,12 @@ from sqlalchemy.types import TypeDecorator
|
||||
# fixed offset is exact and avoids depending on the `tzdata` package (which the
|
||||
# stdlib `zoneinfo` needs on Windows / minimal containers).
|
||||
BEIJING_TZ = timezone(timedelta(hours=8))
|
||||
_POSTGRES_FAMILY_DIALECTS = frozenset({"postgresql", "opengauss"})
|
||||
|
||||
|
||||
def _is_postgres_family(dialect) -> bool:
|
||||
"""Return whether *dialect* preserves timezone-aware timestamps natively."""
|
||||
return dialect.name in _POSTGRES_FAMILY_DIALECTS
|
||||
|
||||
|
||||
class PortableJSON(TypeDecorator):
|
||||
@@ -87,17 +93,17 @@ class BeijingDateTime(TypeDecorator):
|
||||
def process_bind_param(self, value: Any, dialect) -> Any:
|
||||
if not isinstance(value, datetime):
|
||||
return value
|
||||
if dialect.name == "postgresql":
|
||||
if _is_postgres_family(dialect):
|
||||
return value
|
||||
if value.tzinfo is None:
|
||||
# Naive values follow the codebase convention of being UTC.
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
value = value.replace(tzinfo=UTC)
|
||||
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
|
||||
|
||||
def process_result_value(self, value: Any, dialect) -> Any:
|
||||
if not isinstance(value, datetime):
|
||||
return value
|
||||
if dialect.name == "postgresql":
|
||||
if _is_postgres_family(dialect):
|
||||
return value
|
||||
if value.tzinfo is None:
|
||||
# Stored as naive Beijing wall-clock; re-attach the zone.
|
||||
|
||||
@@ -52,8 +52,12 @@ postgres = [
|
||||
"psycopg[binary]>=3.3.3",
|
||||
"psycopg-pool>=3.3.0",
|
||||
]
|
||||
mysql = ["asyncmy>=0.2.10"]
|
||||
pymupdf = ["pymupdf4llm>=0.0.17"]
|
||||
mysql = ["asyncmy>=0.2.10"]
|
||||
gauss = [
|
||||
"asyncpg>=0.29",
|
||||
"opengauss-sqlalchemy>=2.4,<3",
|
||||
]
|
||||
pymupdf = ["pymupdf4llm>=0.0.17"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
|
||||
@@ -30,6 +30,7 @@ dependencies = [
|
||||
[project.optional-dependencies]
|
||||
postgres = ["deerflow-harness[postgres]"]
|
||||
mysql = ["deerflow-harness[mysql]"]
|
||||
gauss = ["deerflow-harness[gauss]"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
|
||||
@@ -22,6 +22,18 @@ class _FakeThreadStore:
|
||||
return self.rows[offset : offset + limit]
|
||||
|
||||
|
||||
class _FakeAgentStore:
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict]:
|
||||
assert user_id == "owner"
|
||||
agents = {
|
||||
"analysis-agent": {
|
||||
"id": "analysis-agent",
|
||||
"name": "舆情分析智能体",
|
||||
}
|
||||
}
|
||||
return [agents[agent_id] for agent_id in agent_ids if agent_id in agents]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: Path, monkeypatch):
|
||||
thread_one = tmp_path / "thread-one" / "outputs"
|
||||
@@ -79,6 +91,7 @@ async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: P
|
||||
|
||||
assert [item["name"] for item in items] == ["证据.zip", "结论.md", "数据汇总.xlsx", "研判报告.docx"]
|
||||
assert [item["file_kind"] for item in items] == ["archive", "text_data", "spreadsheet", "document"]
|
||||
assert [item["source_type"] for item in items] == ["general_qa", "general_qa", "agent", "agent"]
|
||||
assert items[-1]["thread_title"] == "战场环境综合研判"
|
||||
assert items[-1]["agent_id"] == "analysis-agent"
|
||||
assert all(item["name"] not in {"配图.png", "录音.mp3"} for item in items)
|
||||
@@ -88,9 +101,12 @@ async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: P
|
||||
@pytest.mark.asyncio
|
||||
async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Path, monkeypatch):
|
||||
output_dir = tmp_path / "thread-one" / "outputs"
|
||||
agent_output_dir = tmp_path / "thread-agent" / "outputs"
|
||||
output_dir.mkdir(parents=True)
|
||||
agent_output_dir.mkdir(parents=True)
|
||||
(output_dir / "任务报告.md").write_text("report", encoding="utf-8")
|
||||
(output_dir / "任务数据.csv").write_text("a,b", encoding="utf-8")
|
||||
(agent_output_dir / "智能体结论.md").write_text("agent report", encoding="utf-8")
|
||||
|
||||
store = _FakeThreadStore(
|
||||
[
|
||||
@@ -98,7 +114,12 @@ async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Pa
|
||||
"thread_id": "thread-one",
|
||||
"display_name": "任务研判对话",
|
||||
"metadata": {},
|
||||
}
|
||||
},
|
||||
{
|
||||
"thread_id": "thread-agent",
|
||||
"display_name": "智能体研判对话",
|
||||
"metadata": {"agent_id": "analysis-agent"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
@@ -113,6 +134,7 @@ async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Pa
|
||||
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
|
||||
monkeypatch.setattr(artifacts, "get_effective_user_id", lambda: "owner")
|
||||
monkeypatch.setattr(deps, "get_thread_store", lambda _request: store)
|
||||
monkeypatch.setattr(deps, "get_agent_store", lambda _request: _FakeAgentStore())
|
||||
artifacts._artifact_library_cache.clear()
|
||||
|
||||
raw_endpoint = artifacts.list_artifact_library.__wrapped__
|
||||
@@ -120,6 +142,7 @@ async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Pa
|
||||
request=SimpleNamespace(),
|
||||
query="研判对话",
|
||||
file_kind="text_data",
|
||||
source_type="general_qa",
|
||||
offset=0,
|
||||
limit=1,
|
||||
refresh=True,
|
||||
@@ -134,9 +157,24 @@ async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Pa
|
||||
request=SimpleNamespace(),
|
||||
query="任务数据",
|
||||
file_kind="spreadsheet",
|
||||
source_type="all",
|
||||
offset=0,
|
||||
limit=50,
|
||||
refresh=False,
|
||||
)
|
||||
assert spreadsheet_response.total == 1
|
||||
assert spreadsheet_response.items[0].name == "任务数据.csv"
|
||||
|
||||
agent_response = await raw_endpoint(
|
||||
request=SimpleNamespace(),
|
||||
query=None,
|
||||
file_kind="all",
|
||||
source_type="agent",
|
||||
offset=0,
|
||||
limit=50,
|
||||
refresh=False,
|
||||
)
|
||||
assert agent_response.total == 1
|
||||
assert agent_response.items[0].source_type == "agent"
|
||||
assert agent_response.items[0].agent_name == "舆情分析智能体"
|
||||
assert agent_response.items[0].name == "智能体结论.md"
|
||||
@@ -11,6 +11,13 @@ from deerflow.persistence.assistant_knowledge.model import (
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class FakeEmbeddingClient:
|
||||
dimensions = 3
|
||||
|
||||
async def embed_query(self, _query: str) -> list[float]:
|
||||
return [1.0, 0.0, 0.0]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repository():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
@@ -94,3 +101,93 @@ async def test_delete_source_keeps_entity_until_last_contribution_is_removed(
|
||||
async with repository._sf() as session:
|
||||
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
|
||||
assert entity.status == "deleted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_base_import_is_also_mirrored_to_global_base(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
global_base = await repository.initialize(created_by="admin")
|
||||
custom_base = await repository.create_base(
|
||||
created_by="admin",
|
||||
name="专题助手库",
|
||||
description="只承接某个专题的 WeKnora 导出数据",
|
||||
)
|
||||
package = {
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [{"slug": "wiki/topic", "title": "专题 Wiki", "content": "王五负责统筹。"}],
|
||||
"entities": [{"local_id": "e1", "type": "person", "name": "王五"}],
|
||||
"relations": [],
|
||||
}
|
||||
|
||||
job = await repository.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key="kb-topic",
|
||||
source_name="专题普通库",
|
||||
trigger="manual_from_weknora",
|
||||
created_by="admin",
|
||||
mapping_id="kb-topic",
|
||||
base_id=custom_base["id"],
|
||||
)
|
||||
|
||||
assert job["base_id"] == custom_base["id"]
|
||||
assert job["mirrored_global_base_id"] == global_base["id"]
|
||||
assert len(await repository.list_pages(base_id=custom_base["id"], query="王五")) == 2
|
||||
assert len(await repository.list_pages(base_id=global_base["id"], query="王五")) == 2
|
||||
bases = await repository.list_bases()
|
||||
assert [base["id"] for base in bases[:2]] == [global_base["id"], custom_base["id"]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_imported_vectors_are_reused_for_assistant_search(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
job = await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [{"slug": "wiki/vector", "title": "向量页", "content": "页面正文"}],
|
||||
"vectors": [
|
||||
{
|
||||
"wiki_slug": "wiki/vector",
|
||||
"section_index": 0,
|
||||
"section_content": "张三负责向量化检索。",
|
||||
"embedding_model": "bge-embedding-m3",
|
||||
"embedding_dimensions": 3,
|
||||
"vector": [1.0, 0.0, 0.0],
|
||||
},
|
||||
{
|
||||
"wiki_slug": "wiki/vector",
|
||||
"section_index": 1,
|
||||
"section_content": "无关片段。",
|
||||
"embedding_model": "bge-embedding-m3",
|
||||
"embedding_dimensions": 3,
|
||||
"vector": [0.0, 1.0, 0.0],
|
||||
},
|
||||
],
|
||||
},
|
||||
source_type="weknora",
|
||||
source_key="kb-vector",
|
||||
source_name="向量普通库",
|
||||
trigger="manual_package_upload",
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
assert job["counts"]["vectors"] == 2
|
||||
results = await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="这个查询不靠关键词",
|
||||
embedding_client=FakeEmbeddingClient(),
|
||||
)
|
||||
assert results[0]["retrieval_mode"] == "vector"
|
||||
assert results[0]["matched_section"]["embedding_model"] == "bge-embedding-m3"
|
||||
assert results[0]["content_markdown"] == "张三负责向量化检索。"
|
||||
|
||||
sources = await repository.list_sources()
|
||||
assert await repository.delete_source(sources[0]["id"])
|
||||
assert await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="这个查询不靠关键词",
|
||||
embedding_client=FakeEmbeddingClient(),
|
||||
) == []
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Regression coverage for the openGauss/GaussDB persistence backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.schema import CreateIndex, CreateTable
|
||||
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[
|
||||
(
|
||||
"opengauss://user:pass@127.0.0.1:26000/deerflow",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
),
|
||||
(
|
||||
"postgresql://user:pass@127.0.0.1:26000/deerflow?ssl=require",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow?ssl=require",
|
||||
),
|
||||
(
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_gauss_urls_always_select_the_opengauss_async_dialect(
|
||||
configured: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
config = DatabaseConfig(backend="gauss", gauss_url=configured)
|
||||
assert config.app_sqlalchemy_url == expected
|
||||
|
||||
|
||||
def test_gauss_backend_requires_a_supported_nonempty_url() -> None:
|
||||
with pytest.raises(ValueError, match="gauss_url is required"):
|
||||
_ = DatabaseConfig(backend="gauss").app_sqlalchemy_url
|
||||
|
||||
with pytest.raises(ValueError, match="must use opengauss"):
|
||||
_ = DatabaseConfig(
|
||||
backend="gauss",
|
||||
gauss_url="mysql://user:pass@127.0.0.1/deerflow",
|
||||
).app_sqlalchemy_url
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
DatabaseConfig(backend="unknown")
|
||||
|
||||
|
||||
def test_gauss_timestamps_keep_timezone_information() -> None:
|
||||
dialect = type("GaussDialect", (), {"name": "opengauss"})()
|
||||
value = datetime(2026, 9, 2, 8, 30, tzinfo=UTC)
|
||||
column_type = BeijingDateTime()
|
||||
|
||||
assert column_type.process_bind_param(value, dialect) is value
|
||||
assert column_type.process_result_value(value, dialect) is value
|
||||
|
||||
|
||||
def test_official_async_dialect_compiles_the_complete_orm_schema() -> None:
|
||||
"""Every registered table/index must compile before startup touches GaussDB."""
|
||||
pytest.importorskip("opengauss_sqlalchemy")
|
||||
|
||||
import deerflow.persistence.models # noqa: F401 -- register ORM metadata
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
dialect_cls = make_url("opengauss+asyncpg://user:pass@localhost/db").get_dialect(
|
||||
_is_async=True
|
||||
)
|
||||
dialect = dialect_cls()
|
||||
|
||||
assert dialect.__class__.__module__.startswith("opengauss_sqlalchemy")
|
||||
assert Base.metadata.sorted_tables
|
||||
for table in Base.metadata.sorted_tables:
|
||||
assert str(CreateTable(table).compile(dialect=dialect)).startswith("\nCREATE TABLE")
|
||||
for index in table.indexes:
|
||||
assert str(CreateIndex(index).compile(dialect=dialect)).startswith("CREATE")
|
||||
@@ -66,6 +66,9 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
|
||||
assert store.write_batch_size == 1
|
||||
assert len(snapshot["rows"]) == 3
|
||||
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
|
||||
streamed = [row async for row in store.iter_vector_snapshot_rows("mapping-1", "fp", batch_size=2)]
|
||||
assert [row["section_index"] for row in streamed] == [0, 1, 2]
|
||||
assert streamed[0]["page"]["slug"] == "people/alice"
|
||||
page_statuses = await store.list_page_index_status("mapping-1")
|
||||
assert len(page_statuses) == 1
|
||||
assert page_statuses[0] == {
|
||||
|
||||
Reference in new issue
Block a user