feat: integrate WeKnora llmwiki knowledge flow
This commit is contained in:
1 parent
a0a75a9964
commit
e0c09f6704
63 files changed
+7045
-197
No files matched your search
@@ -13,7 +13,6 @@ from deerflow.config.extensions_config import (
|
||||
get_extensions_config,
|
||||
)
|
||||
|
||||
|
||||
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
|
||||
|
||||
|
||||
@@ -27,7 +26,11 @@ def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
if _message_type(messages[cursor]) == "human":
|
||||
end = cursor
|
||||
break
|
||||
sources = _collect_sources(messages[index + 1 : end])
|
||||
preloaded_sources = _collect_preloaded_sources(message, start_index=1)
|
||||
sources = [
|
||||
*preloaded_sources,
|
||||
*_collect_sources(messages[index + 1 : end], start_index=len(preloaded_sources) + 1),
|
||||
]
|
||||
if sources:
|
||||
anchor = _find_reference_batch_anchor(messages, index + 1, end)
|
||||
message_id = _message_id(anchor) or _message_id(message) or str(index)
|
||||
@@ -46,7 +49,11 @@ def build_latest_reference_batch(messages: list[Any]) -> dict[str, Any] | None:
|
||||
human_index = _find_latest_human_index(messages)
|
||||
if human_index is None:
|
||||
return None
|
||||
sources = _collect_sources(messages[human_index + 1 :])
|
||||
preloaded_sources = _collect_preloaded_sources(messages[human_index], start_index=1)
|
||||
sources = [
|
||||
*preloaded_sources,
|
||||
*_collect_sources(messages[human_index + 1 :], start_index=len(preloaded_sources) + 1),
|
||||
]
|
||||
if not sources:
|
||||
return None
|
||||
assistant = _find_reference_batch_anchor(messages, human_index + 1, len(messages))
|
||||
@@ -365,7 +372,7 @@ def _source_debug_item(source: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
def _collect_sources(messages: list[Any], *, start_index: int = 1) -> list[dict[str, Any]]:
|
||||
sources: list[dict[str, Any]] = []
|
||||
pending_tool_contexts: list[dict[str, Any]] = []
|
||||
tool_contexts_by_id: dict[str, dict[str, Any]] = {}
|
||||
@@ -393,7 +400,7 @@ def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
)
|
||||
if configured_sources is not None:
|
||||
for normalized in configured_sources:
|
||||
normalized["index"] = len(sources) + 1
|
||||
normalized["index"] = start_index + len(sources)
|
||||
sources.append(normalized)
|
||||
continue
|
||||
for raw_item in _first_list(payload):
|
||||
@@ -405,11 +412,37 @@ def _collect_sources(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
)
|
||||
if not normalized:
|
||||
continue
|
||||
normalized["index"] = len(sources) + 1
|
||||
normalized["index"] = start_index + len(sources)
|
||||
sources.append(normalized)
|
||||
return sources
|
||||
|
||||
|
||||
def _collect_preloaded_sources(message: Any, *, start_index: int = 1) -> list[dict[str, Any]]:
|
||||
extra = _message_additional_kwargs(message)
|
||||
raw_sources = extra.get("llmwiki_reference_sources")
|
||||
if not isinstance(raw_sources, list):
|
||||
return []
|
||||
sources: list[dict[str, Any]] = []
|
||||
for raw in raw_sources:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
source = dict(raw)
|
||||
index = start_index + len(sources)
|
||||
source["index"] = index
|
||||
source.setdefault("skillName", "llmwiki_search")
|
||||
source.setdefault("mode", "citation")
|
||||
source.setdefault("type", "llmwiki")
|
||||
title = str(source.get("title") or "").strip()
|
||||
if not title:
|
||||
source["title"] = f"LLMWiki 片段 {index}"
|
||||
snippet = str(source.get("snippet") or source.get("content") or "").strip()
|
||||
source["snippet"] = snippet[:360]
|
||||
if "raw" not in source or not isinstance(source.get("raw"), dict):
|
||||
source["raw"] = {key: value for key, value in source.items() if key != "raw"}
|
||||
sources.append(source)
|
||||
return sources
|
||||
|
||||
|
||||
def _collect_configured_sources(
|
||||
payload: Any,
|
||||
fallback_index: int,
|
||||
@@ -486,6 +519,14 @@ def _message_content(message: Any) -> Any:
|
||||
return message.get("content") if isinstance(message, dict) else getattr(message, "content", None)
|
||||
|
||||
|
||||
def _message_additional_kwargs(message: Any) -> dict[str, Any]:
|
||||
if isinstance(message, dict):
|
||||
value = message.get("additional_kwargs") or message.get("additionalKwargs")
|
||||
else:
|
||||
value = getattr(message, "additional_kwargs", None)
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _message_name(message: Any) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("name") or (message.get("additional_kwargs") or {}).get("name") or "tool")
|
||||
|
||||
Reference in new issue
Block a user