sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -2,8 +2,10 @@
from __future__ import annotations
import hashlib
import json
import re
import unicodedata
from typing import Any
from urllib.parse import quote
@@ -14,6 +16,15 @@ from deerflow.config.extensions_config import (
)
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
_SMART_REFERENCE_TOOL_NAMES = frozenset(
{
"ordinary_knowledge_search",
"ordinary_chunk_regex_search",
"ordinary_document_chunks",
"ordinary_wiki_search",
"ordinary_wiki_read_page",
}
)
def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
@@ -27,10 +38,16 @@ def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
end = cursor
break
preloaded_sources = _collect_preloaded_sources(message, start_index=1)
sources = [
*preloaded_sources,
*_collect_sources(messages[index + 1 : end], start_index=len(preloaded_sources) + 1),
]
sources = _dedupe_and_reindex_sources(
[
*preloaded_sources,
*_collect_sources(
messages[index + 1 : end],
start_index=len(preloaded_sources) + 1,
),
],
start_index=1,
)
if sources:
anchor = _find_reference_batch_anchor(messages, index + 1, end)
message_id = _message_id(anchor) or _message_id(message) or str(index)
@@ -50,10 +67,16 @@ def build_latest_reference_batch(messages: list[Any]) -> dict[str, Any] | None:
if human_index is None:
return None
preloaded_sources = _collect_preloaded_sources(messages[human_index], start_index=1)
sources = [
*preloaded_sources,
*_collect_sources(messages[human_index + 1 :], start_index=len(preloaded_sources) + 1),
]
sources = _dedupe_and_reindex_sources(
[
*preloaded_sources,
*_collect_sources(
messages[human_index + 1 :],
start_index=len(preloaded_sources) + 1,
),
],
start_index=1,
)
if not sources:
return None
assistant = _find_reference_batch_anchor(messages, human_index + 1, len(messages))
@@ -956,7 +979,20 @@ def _normalize_reference_item(
return configured
layers = _nested_records(raw)
title = _pick(layers, ("title", "m_title", "name", "question", "keyword", "query", "text"))
content = _pick(layers, ("content", "page_content", "summary", "snippet", "description", "content_preview", "abstract", "body"))
content = _pick(
layers,
(
"content_markdown",
"content",
"page_content",
"summary",
"snippet",
"description",
"content_preview",
"abstract",
"body",
),
)
url = _pick(layers, ("url", "link", "href", "source_url", "web_url"))
source = _pick(layers, ("source", "source1", "site", "provider", "from"))
author = _pick(layers, ("author", "authors", "creator", "writer", "byline", "publisher", "owner"))
@@ -1001,3 +1037,224 @@ def _reference_key(source: dict[str, Any]) -> str:
str(source.get(key) or "").casefold().strip()
for key in ("title", "source", "time", "content", "snippet")
)
def _normalized_identity_text(value: Any) -> str:
text = unicodedata.normalize("NFKC", str(value or "")).casefold()
# Retrieval providers may wrap the same Chinese sentence at different
# positions. Whitespace is not semantic for the exact-content fallback;
# title and knowledge-base scope remain part of the identity.
return re.sub(r"\s+", "", text)
def _source_identity_aliases(source: dict[str, Any]) -> set[str]:
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
mapping_id = str(
raw.get("knowledge_base_id")
or raw.get("mapping_id")
or raw.get("knowledgeBaseId")
or ""
).strip()
scope = mapping_id or _normalized_identity_text(source.get("source")) or "global"
skill_name = str(source.get("skillName") or "").casefold()
result_set_key = _normalized_identity_text(source.get("resultSetKey"))
page_ref = str(raw.get("page_ref") or raw.get("pageRef") or "").strip()
wiki_slug = str(
raw.get("slug") or raw.get("wiki_slug") or raw.get("wikiSlug") or ""
).strip()
chunk_id = str(raw.get("chunk_id") or raw.get("chunkId") or "").strip()
is_wiki_read = "wiki_read_page" in skill_name or bool(
raw.get("content_markdown")
)
raw_offset = raw.get("content_offset", raw.get("contentOffset"))
content_offset = (
raw_offset
if isinstance(raw_offset, int) and not isinstance(raw_offset, bool)
else 0
)
aliases: set[str] = set()
if page_ref:
if is_wiki_read:
aliases.add(
f"wiki:{scope}:page-ref:{page_ref}:offset:{content_offset}"
)
if content_offset == 0:
aliases.add(f"wiki:{scope}:page-ref:{page_ref}:lead")
else:
aliases.add(f"wiki:{scope}:page-ref:{page_ref}:lead")
if wiki_slug:
if is_wiki_read:
aliases.add(f"wiki:{scope}:slug:{wiki_slug}:offset:{content_offset}")
if content_offset == 0:
aliases.add(f"wiki:{scope}:slug:{wiki_slug}:lead")
else:
aliases.add(f"wiki:{scope}:slug:{wiki_slug}:lead")
if chunk_id:
aliases.add(f"raw:{scope}:chunk:{chunk_id}")
has_document_id = bool(
raw.get("knowledge_id")
or raw.get("document_id")
or raw.get("knowledgeId")
or raw.get("documentId")
)
url = str(source.get("url") or raw.get("url") or "").strip().casefold()
if url and not chunk_id and not has_document_id:
aliases.add(f"url:{url}")
item_id = str(source.get("id") or "").strip().casefold()
# Knowledge document ids identify a whole file and therefore must not
# collapse its different chunks. Generic skill/web ids remain useful.
if item_id and not has_document_id and not chunk_id:
skill_scope = _normalized_identity_text(skill_name) or "backend-reference"
aliases.add(
f"id:{skill_scope}:{result_set_key or 'default'}:{scope}:{item_id}"
)
title = _normalized_identity_text(source.get("title"))
content = _normalized_identity_text(
source.get("content") or source.get("snippet")
)
if title and len(content) >= 12:
digest = hashlib.sha256(f"{title}\0{content}".encode()).hexdigest()
if page_ref or wiki_slug or "wiki" in skill_name:
surface = (
f"wiki-read:{content_offset}" if is_wiki_read else "wiki-search"
)
elif skill_name in _SMART_REFERENCE_TOOL_NAMES:
surface = "raw"
else:
skill_scope = _normalized_identity_text(skill_name) or "backend-reference"
surface = f"skill:{skill_scope}:{result_set_key or 'default'}"
aliases.add(f"{surface}:{scope}:content:{digest}")
return aliases
def _source_quality(source: dict[str, Any], sequence: int) -> tuple[int, int, int, float, int]:
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
skill_name = str(source.get("skillName") or "").casefold()
is_wiki_read = int(
"wiki_read_page" in skill_name or bool(raw.get("content_markdown"))
)
is_document_read = int("document_chunks" in skill_name)
content_length = len(str(source.get("content") or source.get("snippet") or ""))
try:
score = float(source.get("score")) if source.get("score") is not None else float("-inf")
except (TypeError, ValueError):
score = float("-inf")
# Earlier source wins a complete tie so ordering is deterministic.
return (is_wiki_read, is_document_read, content_length, score, -sequence)
def _merge_source_metadata(
preferred: dict[str, Any],
group: list[dict[str, Any]],
) -> dict[str, Any]:
merged = dict(preferred)
for key in (
"id",
"skillName",
"mode",
"title",
"type",
"snippet",
"content",
"source",
"author",
"time",
"url",
"score",
"tableColumns",
"resultSetKey",
"resultSetLabel",
"raw",
):
if merged.get(key) not in (None, "", [], {}):
continue
for candidate in group:
value = candidate.get(key)
if value not in (None, "", [], {}):
merged[key] = value
break
return merged
def _dedupe_and_reindex_sources(
sources: list[dict[str, Any]],
*,
start_index: int,
) -> list[dict[str, Any]]:
"""Deduplicate a whole user-turn batch across every retrieval call.
Provider ids are preferred, with an exact normalized title/content alias
for providers that remap chunk ids between vector queries. Same-title
chunks with different text stay distinct. A later Wiki page read replaces
the corresponding search-summary card while retaining the first card's
stable position.
"""
if len(sources) < 2:
return [
{**source, "index": start_index + offset}
for offset, source in enumerate(sources)
]
parents = list(range(len(sources)))
def find(index: int) -> int:
root = index
while parents[root] != root:
root = parents[root]
while parents[index] != index:
next_index = parents[index]
parents[index] = root
index = next_index
return root
def union(left: int, right: int) -> None:
left_root = find(left)
right_root = find(right)
if left_root != right_root:
parents[right_root] = left_root
alias_owner: dict[str, int] = {}
aliases_by_index: list[set[str]] = []
for index, source in enumerate(sources):
aliases = _source_identity_aliases(source)
aliases_by_index.append(aliases)
for alias in aliases:
existing = alias_owner.get(alias)
if existing is None:
alias_owner[alias] = index
else:
union(existing, index)
groups: dict[int, list[int]] = {}
for index, aliases in enumerate(aliases_by_index):
if aliases:
groups.setdefault(find(index), []).append(index)
hidden: set[int] = set()
replacements: dict[int, dict[str, Any]] = {}
for indexes in groups.values():
if len(indexes) < 2:
continue
owner = min(indexes)
preferred_index = max(
indexes,
key=lambda index: _source_quality(sources[index], index),
)
group = [sources[index] for index in indexes]
replacements[owner] = _merge_source_metadata(
sources[preferred_index],
group,
)
hidden.update(index for index in indexes if index != owner)
deduped: list[dict[str, Any]] = []
for index, source in enumerate(sources):
if index in hidden:
continue
item = replacements.get(index, source)
deduped.append({**item, "index": start_index + len(deduped)})
return deduped