sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -1,6 +1,6 @@
from .tools import get_available_tools
from .tools import get_available_tools, get_available_tools_async
__all__ = ["get_available_tools", "skill_manage_tool"]
__all__ = ["get_available_tools", "get_available_tools_async", "skill_manage_tool"]
def __getattr__(name: str):
@@ -713,7 +713,11 @@ async def deep_research_report_tool(
workbench_report_config = _coerce_workbench_report_config(runtime_context.get("writing_report_config"))
workbench_session_id = str(runtime_context.get("workbench_session_id") or "").strip()
force_no_materials = bool(runtime_context.get("force_no_materials"))
harvest_options = {"include_rag_sources": True} if project_context.get("mode") == "solution" else {}
harvest_options = (
{"include_rag_sources": True}
if project_context.get("mode") == "solution" or workbench_session_id
else {}
)
if force_no_materials:
harvested, dropped_count, harvested_count = [], 0, 0
materials: list[ResearchMaterial] = []
@@ -24,6 +24,7 @@ from langgraph.typing import ContextT
from deerflow.agents.thread_state import ThreadState
from deerflow.config.agents_config import load_agent_config
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.knowledge_profile import (
resolved_retrieval_settings,
)
@@ -47,6 +48,9 @@ _MAX_REGEX_CHARS = 256
_MAX_REGEX_SCAN_CHARS = 50_000
_MAX_REGEX_CANDIDATES = 500
_MAX_RESULTS = 30
_MAX_PROVIDER_WIKI_CANDIDATES = 50
_MAX_PROVIDER_WIKI_CONCURRENCY = 4
_PROVIDER_WIKI_REQUEST_TIMEOUT_SECONDS = 5.0
_MAX_TOOL_OUTPUT_CHARS = 48_000
_MAX_PROGRESS_RESULT_ITEMS = 20
_MAX_PROGRESS_RAW_CONTENT_CHARS = 1_600
@@ -211,6 +215,14 @@ def _public_result_details(payload: Mapping[str, Any]) -> dict[str, Any]:
value = payload.get(key)
if isinstance(value, int) and not isinstance(value, bool):
details[key] = max(0, value)
native_regex_supported = payload.get("native_regex_supported")
if isinstance(native_regex_supported, bool):
details["native_regex_supported"] = native_regex_supported
unsupported_count = payload.get("unsupported_knowledge_base_count")
if isinstance(unsupported_count, int) and not isinstance(
unsupported_count, bool
):
details["unsupported_knowledge_base_count"] = max(0, unsupported_count)
channel = str(payload.get("channel") or "").strip()
if channel:
details["channel"] = channel[:80]
@@ -424,13 +436,16 @@ async def _run_with_progress(
)
result_details["result_items"] = result_items
result_details["result_items_truncated"] = result_items_truncated
result_count = int(result_details.get("result_count") or 0)
_emit_progress(
writer,
operation=operation,
stage=stage,
status="completed",
message=f"{completed_message},找到 {result_count} 条结果",
# Keep the human-readable message count-neutral. ``result_count``
# is the sole structured source of truth consumed by the UI; a
# count embedded in this text can become stale when a later frame
# for the same retrieval replaces or filters the result payload.
message=completed_message,
retrieval_id=retrieval_id,
knowledge_base_count=selected_count,
elapsed_ms=elapsed_ms,
@@ -529,7 +544,7 @@ def _assistant_ids(context: Mapping[str, Any]) -> list[str]:
def _allowed_agent_ids(context: Mapping[str, Any]) -> set[str] | None:
agent_id = str(context.get("agent_id") or "").strip()
if not agent_id or agent_id in {"default", "lead_agent"}:
if not agent_id or agent_id in {"default", "lead_agent", "deep-research-planner", "deep-research-collector"}:
return None
try:
config = load_agent_config(agent_id)
@@ -786,48 +801,6 @@ def _complete_local_mirror(state: Mapping[str, Any] | None) -> bool:
return remote_count > 0 and failed_count == 0 and local_count == remote_count
def _regex_literal_tokens(pattern: str) -> list[str]:
"""Extract conservative literal hints without changing regex semantics.
WeKnora executes Wiki search in its database with ``~*``. DeerFlow's
cross-database mirror applies the final expression in Python, but uses
these required-ish literal runs to bring older matching pages into the
bounded candidate set. Empty/fully-structural expressions simply fall
back to the normal newest-page scan.
"""
value = str(pattern or "")
tokens: list[str] = []
current: list[str] = []
escaped = False
def flush() -> None:
token = "".join(current).strip()
current.clear()
if len(token) >= 2 and token not in tokens:
tokens.append(token)
for char in value:
if escaped:
# Character-class escapes (\d, \s, ...) are structural. Escaped
# punctuation (\+, \.) is literal and useful as a search hint.
if char not in "AbBdDsSwWZzG0123456789":
current.append(char)
else:
flush()
escaped = False
continue
if char == "\\":
escaped = True
continue
if char in ".^$*+?{}[]()|":
flush()
continue
current.append(char)
flush()
return sorted(tokens, key=lambda item: (-len(item), item.casefold()))[:12]
def _normalized_wiki_page(mapping_id: str, page: Mapping[str, Any]) -> dict[str, Any]:
"""Normalize provider/local Wiki shapes for identical regex behavior."""
@@ -850,22 +823,6 @@ def _normalized_wiki_page(mapping_id: str, page: Mapping[str, Any]) -> dict[str,
return {**raw, **normalized}
def _merge_wiki_pages(*groups: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]:
merged: list[dict[str, Any]] = []
seen: set[str] = set()
for group in groups:
for page in group:
slug = str(page.get("slug") or page.get("wiki_slug") or page.get("path") or "").strip()
key = slug.casefold() or str(page.get("id") or page.get("page_id") or "").strip()
if not key or key in seen:
continue
seen.add(key)
merged.append(page)
if len(merged) >= limit:
return merged
return merged
def _wiki_search_fields(page: Mapping[str, Any]) -> dict[str, str]:
slug = str(page.get("slug") or page.get("wiki_slug") or page.get("path") or "")
return {
@@ -1262,98 +1219,50 @@ async def _document_chunks_impl(
async def _remote_wiki_pages(
mapping: Mapping[str, Any],
*,
pattern: str,
limit: int,
query_hints: list[str] | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], bool]:
client = _client()
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], Literal["searched", "unsupported", "failed"]]:
mapping_id = str(mapping.get("id") or "")
remote_id = str(mapping.get("weknora_id") or "")
if not remote_id:
return [], [_public_warning("WIKI_SURFACE_UNAVAILABLE", mapping)], False
semaphore = asyncio.Semaphore(8)
warnings: list[dict[str, Any]] = []
return [], [_public_warning("WIKI_SURFACE_UNAVAILABLE", mapping)], "failed"
async def load(*, query: str, cap: int) -> tuple[list[dict[str, Any]], bool]:
summaries: list[dict[str, Any]] = []
page = 1
try:
while len(summaries) < cap:
page_size = min(100, cap - len(summaries))
kwargs: dict[str, Any] = {"page": page, "page_size": page_size}
if query:
kwargs["query"] = query
payload = await client.list_wiki_pages(remote_id, **kwargs)
batch = [
dict(item)
for item in payload.get("pages") or []
if isinstance(item, Mapping)
]
summaries.extend(batch)
if not batch or page >= int(payload.get("total_pages") or 1):
break
page += 1
except Exception: # noqa: BLE001 - another provider path may still work
return [], False
return summaries, True
client = _client()
native_search = getattr(client, "search_wiki_pages", None)
if not callable(native_search):
return [], [], "unsupported"
# The live WeKnora list endpoint accepts a query. Probe a small number of
# regex literal runs first so an old matching page is not hidden behind the
# newest-page scan cap, then fill the remaining budget from the unfiltered
# directory. The final POSIX expression is still evaluated locally over
# every returned full page, so provider query behavior cannot create false
# positives.
hints = list(dict.fromkeys(query_hints or []))[:4]
query_cap = max(25, (limit + max(1, len(hints)) - 1) // max(1, len(hints)))
queried: list[dict[str, Any]] = []
any_query_succeeded = False
for hint in hints:
batch, succeeded = await load(query=hint, cap=min(limit, query_cap))
any_query_succeeded = any_query_succeeded or succeeded
queried = _merge_wiki_pages(queried, batch, limit=limit)
recent, recent_succeeded = await load(query="", cap=limit)
if not recent_succeeded and not any_query_succeeded:
return [], [_public_warning("PROVIDER_WIKI_LIST_FAILED", mapping)], False
summaries = _merge_wiki_pages(queried, recent, limit=limit)
async def read(summary: dict[str, Any]) -> dict[str, Any] | None:
slug = str(summary.get("slug") or summary.get("path") or "").strip()
if not slug:
return None
async with semaphore:
detail = await client.get_wiki_page(remote_id, slug)
return {
**summary,
**detail,
"knowledge_base_mapping_id": mapping_id,
}
details = await asyncio.gather(
*(read(summary) for summary in summaries[:limit]),
return_exceptions=True,
)
rows: list[dict[str, Any]] = []
detail_failures = 0
for summary, detail in zip(summaries[:limit], details, strict=True):
if isinstance(detail, Exception):
detail_failures += 1
rows.append(
{
**summary,
"knowledge_base_mapping_id": mapping_id,
}
)
elif isinstance(detail, dict):
rows.append(_normalized_wiki_page(mapping_id, detail))
if detail_failures:
warnings.append(
_public_warning(
"PROVIDER_WIKI_DETAIL_PARTIAL",
mapping,
failed_pages=detail_failures,
)
# Smart Wiki regex search has exactly one provider path. WeKnora's native
# endpoint searches title/slug/summary/content and returns complete pages.
# When an older provider does not expose that endpoint, skip the Wiki
# surface for this turn. Never emulate regex search by enumerating the Wiki
# directory or fetching page bodies one by one.
bounded_limit = min(max(1, int(limit)), _MAX_PROVIDER_WIKI_CANDIDATES)
try:
native_rows = await asyncio.wait_for(
native_search(remote_id, pattern, limit=bounded_limit),
timeout=_PROVIDER_WIKI_REQUEST_TIMEOUT_SECONDS,
)
return rows, warnings, True
except WeKnoraError as exc:
if exc.status_code in {404, 405}:
return [], [], "unsupported"
return [], [_public_warning("PROVIDER_WIKI_SEARCH_FAILED", mapping)], "failed"
except Exception: # noqa: BLE001 - timeout/transport failures are provider failures
return [], [_public_warning("PROVIDER_WIKI_SEARCH_FAILED", mapping)], "failed"
rows = native_rows if isinstance(native_rows, list) else []
return (
[
_normalized_wiki_page(
mapping_id,
{**row, "knowledge_base_mapping_id": mapping_id},
)
for row in rows[:bounded_limit]
if isinstance(row, Mapping)
],
[],
"searched",
)
async def _wiki_index_states(index_store: Any, mapping_ids: list[str]) -> dict[str, dict[str, Any]]:
@@ -1373,107 +1282,80 @@ async def _wiki_index_states(index_store: Any, mapping_ids: list[str]) -> dict[s
async def _wiki_search_impl(pattern: str, context: Mapping[str, Any]) -> str:
expression = _safe_regex(pattern)
literal_tokens = _regex_literal_tokens(pattern)
scope = await _authorized_scope(context)
mapping_by_id = {str(row.get("id") or ""): row for row in scope.mappings}
quotas = _fair_quotas(scope.mapping_ids, _MAX_REGEX_CANDIDATES)
try:
index_store = get_local_wiki_index_store()
states = await _wiki_index_states(index_store, scope.mapping_ids)
except RuntimeError:
index_store = None
states = {}
rows: list[dict[str, Any]] = []
warnings: list[dict[str, Any]] = []
coverage: list[dict[str, Any]] = []
successful_mappings = 0
for mapping_id in scope.mapping_ids:
unsupported_mappings = 0
failed_mappings = 0
scan_truncated = False
semaphore = asyncio.Semaphore(_MAX_PROVIDER_WIKI_CONCURRENCY)
async def search_mapping(
mapping_id: str,
) -> tuple[
str,
list[dict[str, Any]],
list[dict[str, Any]],
Literal["searched", "unsupported", "failed"],
]:
async with semaphore:
mapping_rows, mapping_warnings, status = await _remote_wiki_pages(
mapping_by_id[mapping_id],
pattern=pattern,
limit=quotas.get(mapping_id, 1),
)
return mapping_id, mapping_rows, mapping_warnings, status
# Each selected knowledge base is independent. Bound concurrency so the
# total latency is roughly one provider request instead of N sequential
# request timeouts for N selected libraries.
mapping_results = await asyncio.gather(
*(search_mapping(mapping_id) for mapping_id in scope.mapping_ids)
)
for mapping_id, mapping_rows, mapping_warnings, status in mapping_results:
mapping = mapping_by_id[mapping_id]
quota = quotas.get(mapping_id, 1)
mapping_rows: list[dict[str, Any]] | None = None
channel = "provider"
if index_store is not None and _complete_local_mirror(states.get(mapping_id)):
try:
hinted_rows: list[dict[str, Any]] = []
keyword_search = getattr(index_store, "search_pages_by_keywords", None)
if literal_tokens and callable(keyword_search):
hinted_rows = await keyword_search(
[mapping_id],
literal_tokens,
limit=quota,
)
recent_rows = await index_store.list_pages([mapping_id], limit=quota)
mapping_rows = _merge_wiki_pages(
[
_normalized_wiki_page(mapping_id, row)
for row in hinted_rows
if isinstance(row, Mapping)
],
[
_normalized_wiki_page(mapping_id, row)
for row in recent_rows
if isinstance(row, Mapping)
],
limit=quota,
)
channel = "local_wiki_mirror"
except Exception: # noqa: BLE001 - retry this mapping through provider
warnings.append(
_public_warning(
"LOCAL_WIKI_MIRROR_FAILED_PROVIDER_FALLBACK",
mapping,
)
)
if mapping_rows is not None and not any(
_wiki_hit_fields(expression, page)[1] for page in mapping_rows
):
# A mirror can become stale after a completed scan (for example,
# Wiki generation finishes while auto-sync is disabled). A local
# zero-match must therefore probe the live provider before we tell
# the reasoning loop that the Wiki has no relevant page.
provider_rows, provider_warnings, provider_succeeded = (
await _remote_wiki_pages(
mapping,
limit=quota,
query_hints=literal_tokens,
)
warnings.extend(mapping_warnings)
if status == "failed":
failed_mappings += 1
coverage.append(
{
"knowledge_base": str(mapping.get("name") or ""),
"channel": "provider_native_regex",
"status": "failed",
}
)
warnings.extend(provider_warnings)
if provider_succeeded:
mapping_rows = _merge_wiki_pages(
provider_rows,
mapping_rows,
limit=quota,
)
channel = "local_wiki_mirror_then_provider"
if mapping_rows is None:
mapping_rows, mapping_warnings, succeeded = await _remote_wiki_pages(
mapping,
limit=quota,
query_hints=literal_tokens,
continue
if status == "unsupported":
unsupported_mappings += 1
coverage.append(
{
"knowledge_base": str(mapping.get("name") or ""),
"channel": "provider_native_regex",
"status": "unsupported",
}
)
warnings.extend(mapping_warnings)
if not succeeded:
coverage.append(
{
"knowledge_base": str(mapping.get("name") or ""),
"channel": "provider",
"status": "failed",
}
)
continue
continue
successful_mappings += 1
rows.extend(mapping_rows)
mapping_limit = min(
max(1, int(quotas.get(mapping_id, 1))),
_MAX_PROVIDER_WIKI_CANDIDATES,
)
scan_truncated = scan_truncated or len(mapping_rows) >= mapping_limit
coverage.append(
{
"knowledge_base": str(mapping.get("name") or ""),
"channel": channel,
"channel": "provider_native_regex",
"status": "searched",
}
)
if successful_mappings == 0:
if successful_mappings == 0 and failed_mappings > 0:
return _error(
"wiki_search",
SmartKnowledgeToolError(
@@ -1506,18 +1388,18 @@ async def _wiki_search_impl(pattern: str, context: Mapping[str, Any]) -> str:
key=lambda item: (len(item["matched_fields"]), str(item.get("updated_at") or "")),
reverse=True,
)
channels = {str(item["channel"]) for item in coverage if item["status"] == "searched"}
channel = next(iter(channels)) if len(channels) == 1 else "mixed"
return _result(
operation="wiki_search",
results=matches[:_MAX_RESULTS],
pattern=pattern,
channel=channel,
partial=successful_mappings < len(scope.mapping_ids) or bool(warnings),
channel="provider_native_regex",
partial=failed_mappings > 0 or bool(warnings),
warnings=warnings,
coverage=coverage,
native_regex_supported=successful_mappings > 0,
unsupported_knowledge_base_count=unsupported_mappings,
scanned_pages=len(rows),
scan_truncated=len(rows) >= _MAX_REGEX_CANDIDATES,
scan_truncated=scan_truncated,
)
@@ -157,7 +157,7 @@ def selected_assistant_knowledge_base_ids(context: Mapping[str, Any] | None) ->
def _agent_allowed_ids(context: Mapping[str, Any] | None) -> list[str] | None:
agent_id = str((context or {}).get("agent_id") or "").strip()
if not agent_id or agent_id in {"default", "lead_agent"}:
if not agent_id or agent_id in {"default", "lead_agent", "deep-research-planner", "deep-research-collector"}:
return None
try:
config = load_agent_config(agent_id)
@@ -68,6 +68,16 @@ def _active_html_map_engine() -> str:
return html_map_engine()
_RUNTIME_BASE_PLACEHOLDER = "__DF_HTML_RUNTIME_BASE__"
def _html_runtime_base_url() -> str:
"""Browser-reachable runtime base saved in the settings dialog."""
from deerflow.html_runtime.addresses import html_runtime_base_url
return html_runtime_base_url()
def _visualization_contract(required: set[str]) -> dict[str, object]:
leaflet = _active_html_map_engine() == "leaflet"
map_contract: dict[str, object] = (
@@ -296,6 +306,8 @@ def _prepare(
# usable 16px-or-larger starting point.
template_html = _apply_minimum_font_size(source.read_text(encoding="utf-8"))
template_html = _apply_document_theme(template_html, profile.theme)
runtime_base_url = _html_runtime_base_url()
template_html = template_html.replace(_RUNTIME_BASE_PLACEHOLDER, runtime_base_url)
working_copy.write_text(template_html, encoding="utf-8")
brief = {
"template": profile.to_dict(),
@@ -208,8 +208,11 @@ def _load_owned_skill_names(user_id: str | None) -> set[str] | None:
factory = get_session_factory()
if factory is None:
return None
store = make_skill_store(factory)
records = run_db_blocking(store.list_visible(user_id))
async def read_visible():
store = make_skill_store(get_session_factory())
return await store.list_visible(user_id)
records = run_db_blocking(read_visible())
return {r["name"] for r in records if r.get("owner_user_id") == user_id}
except Exception:
logger.debug("Failed to load owned skill names for skill_list filter", exc_info=True)
@@ -147,7 +147,7 @@ async def task_tool(
# Get available tools (excluding task tool to prevent nesting)
# Lazy import to avoid circular dependency
from deerflow.tools import get_available_tools
from deerflow.tools import get_available_tools_async
# Inherit parent agent's tool_groups so subagents respect the same restrictions
parent_tool_groups = metadata.get("tool_groups")
@@ -157,6 +157,8 @@ async def task_tool(
effective_model = resolve_subagent_model_name(config, parent_model, app_config=resolved_app_config)
# Subagents should not have subagent tools enabled (prevent recursive nesting)
parent_context = runtime.context if runtime is not None and isinstance(getattr(runtime, "context", None), dict) else {}
roundtable_file_stream = bool(parent_context.get("roundtable_write_file_compatibility"))
available_tools_kwargs = {
"model_name": effective_model,
"groups": parent_tool_groups,
@@ -164,7 +166,9 @@ async def task_tool(
}
if resolved_app_config is not None:
available_tools_kwargs["app_config"] = resolved_app_config
tools = get_available_tools(**available_tools_kwargs)
if roundtable_file_stream:
available_tools_kwargs["write_file_compatibility"] = True
tools = await get_available_tools_async(**available_tools_kwargs)
# Create executor
executor_kwargs = {
@@ -178,6 +182,8 @@ async def task_tool(
}
if resolved_app_config is not None:
executor_kwargs["app_config"] = resolved_app_config
if roundtable_file_stream:
executor_kwargs["write_file_compatibility"] = True
executor = SubagentExecutor(**executor_kwargs)
# Start background execution (always async to prevent blocking)
@@ -58,6 +58,41 @@ def _enforce_agent_orchestration_scope(
if allow_agent_orchestration:
return tools
return [tool for tool in tools if tool.name != "agent_orchestration"]
async def get_available_tools_async(
groups: list[str] | None = None,
include_mcp: bool = True,
model_name: str | None = None,
subagent_enabled: bool = False,
is_scheduled_run: bool = False,
excluded_tools: list[str] | None = None,
*,
allowed_tools: list[str] | None = None,
allow_agent_orchestration: bool = False,
app_config: AppConfig | None = None,
write_file_compatibility: bool | None = None,
) -> list[BaseTool]:
"""Load tools without blocking the caller loop or its MCP discovery.
Share bounded initialization with agent construction and restore the
resulting deferred-tool registry into the calling task before execution.
"""
from deerflow.utils.background_init import run_background_init
return await run_background_init(
get_available_tools,
groups=groups,
include_mcp=include_mcp,
model_name=model_name,
subagent_enabled=subagent_enabled,
is_scheduled_run=is_scheduled_run,
excluded_tools=excluded_tools,
allowed_tools=allowed_tools,
allow_agent_orchestration=allow_agent_orchestration,
app_config=app_config,
write_file_compatibility=write_file_compatibility,
)
def get_available_tools(
@@ -68,8 +103,10 @@ def get_available_tools(
is_scheduled_run: bool = False,
excluded_tools: list[str] | None = None,
*,
allowed_tools: list[str] | None = None,
allow_agent_orchestration: bool = False,
app_config: AppConfig | None = None,
write_file_compatibility: bool | None = None,
) -> list[BaseTool]:
"""Get all available tools from config.
@@ -84,6 +121,10 @@ def get_available_tools(
excluded_tools: Optional list of tool names to exclude from the result
(applied AFTER all other resolution). Takes precedence over
``app_config.excluded_tools``.
allowed_tools: Optional request-scoped allowlist applied after all tools
have been resolved. ``None`` keeps the normal tool surface; an empty
list disables every ordinary tool. Exclusions are applied afterwards
and therefore still win.
allow_agent_orchestration: Explicit coordinator-only capability gate. Defaults to
``False`` so ordinary agents and all indirect callers fail closed.
@@ -100,6 +141,8 @@ def get_available_tools(
# requests entirely inside PatchedChatOpenAI.
global_write_file_compatibility_enabled = (
get_write_file_compatibility_settings().enabled
if write_file_compatibility is None
else write_file_compatibility
)
write_renderer_enabled = (
not global_write_file_compatibility_enabled
@@ -339,6 +382,22 @@ def get_available_tools(
if not write_renderer_enabled:
unique_tools = [tool for tool in unique_tools if tool.name != _BEGIN_WRITE_FILE_TOOL_NAME]
# Security-sensitive internal runs (notably the roundtable coordinator)
# must fail closed when new configured/builtin/MCP tools are added. A
# denylist cannot provide that guarantee: every new tool would otherwise be
# exposed until each caller remembered to exclude it.
if allowed_tools is not None:
allowed_set = {str(name).strip() for name in allowed_tools if str(name).strip()}
filtered_tools = [tool for tool in unique_tools if tool.name in allowed_set]
if len(filtered_tools) < len(unique_tools):
logger.info(
"Allowed request tool surface contains %d of %d resolved tool(s): %s",
len(filtered_tools),
len(unique_tools),
sorted(allowed_set),
)
unique_tools = filtered_tools
# Drop tools the caller (or config) wants excluded. Request-level
# ``excluded_tools`` wins; the config-level list is the static fallback.
effective_excluded = excluded_tools if excluded_tools else getattr(config, "excluded_tools", None) or []