sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
@@ -2,8 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
@@ -14,6 +16,15 @@ from deerflow.config.extensions_config import (
|
||||
)
|
||||
|
||||
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
|
||||
_SMART_REFERENCE_TOOL_NAMES = frozenset(
|
||||
{
|
||||
"ordinary_knowledge_search",
|
||||
"ordinary_chunk_regex_search",
|
||||
"ordinary_document_chunks",
|
||||
"ordinary_wiki_search",
|
||||
"ordinary_wiki_read_page",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
@@ -27,10 +38,16 @@ def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
end = cursor
|
||||
break
|
||||
preloaded_sources = _collect_preloaded_sources(message, start_index=1)
|
||||
sources = [
|
||||
*preloaded_sources,
|
||||
*_collect_sources(messages[index + 1 : end], start_index=len(preloaded_sources) + 1),
|
||||
]
|
||||
sources = _dedupe_and_reindex_sources(
|
||||
[
|
||||
*preloaded_sources,
|
||||
*_collect_sources(
|
||||
messages[index + 1 : end],
|
||||
start_index=len(preloaded_sources) + 1,
|
||||
),
|
||||
],
|
||||
start_index=1,
|
||||
)
|
||||
if sources:
|
||||
anchor = _find_reference_batch_anchor(messages, index + 1, end)
|
||||
message_id = _message_id(anchor) or _message_id(message) or str(index)
|
||||
@@ -50,10 +67,16 @@ def build_latest_reference_batch(messages: list[Any]) -> dict[str, Any] | None:
|
||||
if human_index is None:
|
||||
return None
|
||||
preloaded_sources = _collect_preloaded_sources(messages[human_index], start_index=1)
|
||||
sources = [
|
||||
*preloaded_sources,
|
||||
*_collect_sources(messages[human_index + 1 :], start_index=len(preloaded_sources) + 1),
|
||||
]
|
||||
sources = _dedupe_and_reindex_sources(
|
||||
[
|
||||
*preloaded_sources,
|
||||
*_collect_sources(
|
||||
messages[human_index + 1 :],
|
||||
start_index=len(preloaded_sources) + 1,
|
||||
),
|
||||
],
|
||||
start_index=1,
|
||||
)
|
||||
if not sources:
|
||||
return None
|
||||
assistant = _find_reference_batch_anchor(messages, human_index + 1, len(messages))
|
||||
@@ -956,7 +979,20 @@ def _normalize_reference_item(
|
||||
return configured
|
||||
layers = _nested_records(raw)
|
||||
title = _pick(layers, ("title", "m_title", "name", "question", "keyword", "query", "text"))
|
||||
content = _pick(layers, ("content", "page_content", "summary", "snippet", "description", "content_preview", "abstract", "body"))
|
||||
content = _pick(
|
||||
layers,
|
||||
(
|
||||
"content_markdown",
|
||||
"content",
|
||||
"page_content",
|
||||
"summary",
|
||||
"snippet",
|
||||
"description",
|
||||
"content_preview",
|
||||
"abstract",
|
||||
"body",
|
||||
),
|
||||
)
|
||||
url = _pick(layers, ("url", "link", "href", "source_url", "web_url"))
|
||||
source = _pick(layers, ("source", "source1", "site", "provider", "from"))
|
||||
author = _pick(layers, ("author", "authors", "creator", "writer", "byline", "publisher", "owner"))
|
||||
@@ -1001,3 +1037,224 @@ def _reference_key(source: dict[str, Any]) -> str:
|
||||
str(source.get(key) or "").casefold().strip()
|
||||
for key in ("title", "source", "time", "content", "snippet")
|
||||
)
|
||||
|
||||
|
||||
def _normalized_identity_text(value: Any) -> str:
|
||||
text = unicodedata.normalize("NFKC", str(value or "")).casefold()
|
||||
# Retrieval providers may wrap the same Chinese sentence at different
|
||||
# positions. Whitespace is not semantic for the exact-content fallback;
|
||||
# title and knowledge-base scope remain part of the identity.
|
||||
return re.sub(r"\s+", "", text)
|
||||
|
||||
|
||||
def _source_identity_aliases(source: dict[str, Any]) -> set[str]:
|
||||
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
|
||||
mapping_id = str(
|
||||
raw.get("knowledge_base_id")
|
||||
or raw.get("mapping_id")
|
||||
or raw.get("knowledgeBaseId")
|
||||
or ""
|
||||
).strip()
|
||||
scope = mapping_id or _normalized_identity_text(source.get("source")) or "global"
|
||||
skill_name = str(source.get("skillName") or "").casefold()
|
||||
result_set_key = _normalized_identity_text(source.get("resultSetKey"))
|
||||
page_ref = str(raw.get("page_ref") or raw.get("pageRef") or "").strip()
|
||||
wiki_slug = str(
|
||||
raw.get("slug") or raw.get("wiki_slug") or raw.get("wikiSlug") or ""
|
||||
).strip()
|
||||
chunk_id = str(raw.get("chunk_id") or raw.get("chunkId") or "").strip()
|
||||
is_wiki_read = "wiki_read_page" in skill_name or bool(
|
||||
raw.get("content_markdown")
|
||||
)
|
||||
raw_offset = raw.get("content_offset", raw.get("contentOffset"))
|
||||
content_offset = (
|
||||
raw_offset
|
||||
if isinstance(raw_offset, int) and not isinstance(raw_offset, bool)
|
||||
else 0
|
||||
)
|
||||
aliases: set[str] = set()
|
||||
|
||||
if page_ref:
|
||||
if is_wiki_read:
|
||||
aliases.add(
|
||||
f"wiki:{scope}:page-ref:{page_ref}:offset:{content_offset}"
|
||||
)
|
||||
if content_offset == 0:
|
||||
aliases.add(f"wiki:{scope}:page-ref:{page_ref}:lead")
|
||||
else:
|
||||
aliases.add(f"wiki:{scope}:page-ref:{page_ref}:lead")
|
||||
if wiki_slug:
|
||||
if is_wiki_read:
|
||||
aliases.add(f"wiki:{scope}:slug:{wiki_slug}:offset:{content_offset}")
|
||||
if content_offset == 0:
|
||||
aliases.add(f"wiki:{scope}:slug:{wiki_slug}:lead")
|
||||
else:
|
||||
aliases.add(f"wiki:{scope}:slug:{wiki_slug}:lead")
|
||||
if chunk_id:
|
||||
aliases.add(f"raw:{scope}:chunk:{chunk_id}")
|
||||
|
||||
has_document_id = bool(
|
||||
raw.get("knowledge_id")
|
||||
or raw.get("document_id")
|
||||
or raw.get("knowledgeId")
|
||||
or raw.get("documentId")
|
||||
)
|
||||
url = str(source.get("url") or raw.get("url") or "").strip().casefold()
|
||||
if url and not chunk_id and not has_document_id:
|
||||
aliases.add(f"url:{url}")
|
||||
item_id = str(source.get("id") or "").strip().casefold()
|
||||
# Knowledge document ids identify a whole file and therefore must not
|
||||
# collapse its different chunks. Generic skill/web ids remain useful.
|
||||
if item_id and not has_document_id and not chunk_id:
|
||||
skill_scope = _normalized_identity_text(skill_name) or "backend-reference"
|
||||
aliases.add(
|
||||
f"id:{skill_scope}:{result_set_key or 'default'}:{scope}:{item_id}"
|
||||
)
|
||||
|
||||
title = _normalized_identity_text(source.get("title"))
|
||||
content = _normalized_identity_text(
|
||||
source.get("content") or source.get("snippet")
|
||||
)
|
||||
if title and len(content) >= 12:
|
||||
digest = hashlib.sha256(f"{title}\0{content}".encode()).hexdigest()
|
||||
if page_ref or wiki_slug or "wiki" in skill_name:
|
||||
surface = (
|
||||
f"wiki-read:{content_offset}" if is_wiki_read else "wiki-search"
|
||||
)
|
||||
elif skill_name in _SMART_REFERENCE_TOOL_NAMES:
|
||||
surface = "raw"
|
||||
else:
|
||||
skill_scope = _normalized_identity_text(skill_name) or "backend-reference"
|
||||
surface = f"skill:{skill_scope}:{result_set_key or 'default'}"
|
||||
aliases.add(f"{surface}:{scope}:content:{digest}")
|
||||
return aliases
|
||||
|
||||
|
||||
def _source_quality(source: dict[str, Any], sequence: int) -> tuple[int, int, int, float, int]:
|
||||
raw = source.get("raw") if isinstance(source.get("raw"), dict) else {}
|
||||
skill_name = str(source.get("skillName") or "").casefold()
|
||||
is_wiki_read = int(
|
||||
"wiki_read_page" in skill_name or bool(raw.get("content_markdown"))
|
||||
)
|
||||
is_document_read = int("document_chunks" in skill_name)
|
||||
content_length = len(str(source.get("content") or source.get("snippet") or ""))
|
||||
try:
|
||||
score = float(source.get("score")) if source.get("score") is not None else float("-inf")
|
||||
except (TypeError, ValueError):
|
||||
score = float("-inf")
|
||||
# Earlier source wins a complete tie so ordering is deterministic.
|
||||
return (is_wiki_read, is_document_read, content_length, score, -sequence)
|
||||
|
||||
|
||||
def _merge_source_metadata(
|
||||
preferred: dict[str, Any],
|
||||
group: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
merged = dict(preferred)
|
||||
for key in (
|
||||
"id",
|
||||
"skillName",
|
||||
"mode",
|
||||
"title",
|
||||
"type",
|
||||
"snippet",
|
||||
"content",
|
||||
"source",
|
||||
"author",
|
||||
"time",
|
||||
"url",
|
||||
"score",
|
||||
"tableColumns",
|
||||
"resultSetKey",
|
||||
"resultSetLabel",
|
||||
"raw",
|
||||
):
|
||||
if merged.get(key) not in (None, "", [], {}):
|
||||
continue
|
||||
for candidate in group:
|
||||
value = candidate.get(key)
|
||||
if value not in (None, "", [], {}):
|
||||
merged[key] = value
|
||||
break
|
||||
return merged
|
||||
|
||||
|
||||
def _dedupe_and_reindex_sources(
|
||||
sources: list[dict[str, Any]],
|
||||
*,
|
||||
start_index: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Deduplicate a whole user-turn batch across every retrieval call.
|
||||
|
||||
Provider ids are preferred, with an exact normalized title/content alias
|
||||
for providers that remap chunk ids between vector queries. Same-title
|
||||
chunks with different text stay distinct. A later Wiki page read replaces
|
||||
the corresponding search-summary card while retaining the first card's
|
||||
stable position.
|
||||
"""
|
||||
|
||||
if len(sources) < 2:
|
||||
return [
|
||||
{**source, "index": start_index + offset}
|
||||
for offset, source in enumerate(sources)
|
||||
]
|
||||
|
||||
parents = list(range(len(sources)))
|
||||
|
||||
def find(index: int) -> int:
|
||||
root = index
|
||||
while parents[root] != root:
|
||||
root = parents[root]
|
||||
while parents[index] != index:
|
||||
next_index = parents[index]
|
||||
parents[index] = root
|
||||
index = next_index
|
||||
return root
|
||||
|
||||
def union(left: int, right: int) -> None:
|
||||
left_root = find(left)
|
||||
right_root = find(right)
|
||||
if left_root != right_root:
|
||||
parents[right_root] = left_root
|
||||
|
||||
alias_owner: dict[str, int] = {}
|
||||
aliases_by_index: list[set[str]] = []
|
||||
for index, source in enumerate(sources):
|
||||
aliases = _source_identity_aliases(source)
|
||||
aliases_by_index.append(aliases)
|
||||
for alias in aliases:
|
||||
existing = alias_owner.get(alias)
|
||||
if existing is None:
|
||||
alias_owner[alias] = index
|
||||
else:
|
||||
union(existing, index)
|
||||
|
||||
groups: dict[int, list[int]] = {}
|
||||
for index, aliases in enumerate(aliases_by_index):
|
||||
if aliases:
|
||||
groups.setdefault(find(index), []).append(index)
|
||||
|
||||
hidden: set[int] = set()
|
||||
replacements: dict[int, dict[str, Any]] = {}
|
||||
for indexes in groups.values():
|
||||
if len(indexes) < 2:
|
||||
continue
|
||||
owner = min(indexes)
|
||||
preferred_index = max(
|
||||
indexes,
|
||||
key=lambda index: _source_quality(sources[index], index),
|
||||
)
|
||||
group = [sources[index] for index in indexes]
|
||||
replacements[owner] = _merge_source_metadata(
|
||||
sources[preferred_index],
|
||||
group,
|
||||
)
|
||||
hidden.update(index for index in indexes if index != owner)
|
||||
|
||||
deduped: list[dict[str, Any]] = []
|
||||
for index, source in enumerate(sources):
|
||||
if index in hidden:
|
||||
continue
|
||||
item = replacements.get(index, source)
|
||||
deduped.append({**item, "index": start_index + len(deduped)})
|
||||
return deduped
|
||||
Reference in new issue
Block a user