来源新增修改
This commit is contained in:
1 parent
7652db4601
commit
a01d7123d7
6 files changed
+862
-45
No files matched your search
@@ -0,0 +1,429 @@
|
||||
/**
|
||||
* Normalizes retrieval payloads from web_search, intranet skills, MCP tools, etc.
|
||||
* into a single card-friendly shape for the workspace chat UI.
|
||||
*
|
||||
* Parsing is intentionally tolerant: known field aliases first, then key-name
|
||||
* heuristics, then longest-string fallback for body text.
|
||||
*/
|
||||
|
||||
export type SearchResultItem = {
|
||||
title?: string;
|
||||
url?: string;
|
||||
snippet?: string;
|
||||
content?: string;
|
||||
source?: string;
|
||||
published_date?: string;
|
||||
publishedDate?: string;
|
||||
date?: string;
|
||||
};
|
||||
|
||||
const RESULT_LIST_KEYS = [
|
||||
"results",
|
||||
"data",
|
||||
"items",
|
||||
"hits",
|
||||
"documents",
|
||||
"records",
|
||||
"entries",
|
||||
"matches",
|
||||
"chunks",
|
||||
"sources",
|
||||
] as const;
|
||||
|
||||
const WRAPPER_KEYS = ["result", "payload", "response", "output", "data"] as const;
|
||||
|
||||
const NESTED_META_KEYS = ["metadata", "meta", "attrs", "attributes", "properties", "extra"] as const;
|
||||
|
||||
/** Keys ignored when guessing content from arbitrary string fields. */
|
||||
const SKIP_HEURISTIC_KEYS = new Set([
|
||||
"id",
|
||||
"uuid",
|
||||
"recuuid",
|
||||
"score",
|
||||
"relevance_score",
|
||||
"rank",
|
||||
"index",
|
||||
"type",
|
||||
"status",
|
||||
"ok",
|
||||
"success",
|
||||
"error",
|
||||
"query",
|
||||
]);
|
||||
|
||||
const TITLE_ALIASES = [
|
||||
"title",
|
||||
"m_title",
|
||||
"name",
|
||||
"heading",
|
||||
"headline",
|
||||
"subject",
|
||||
"document_title",
|
||||
"doc_title",
|
||||
"file_name",
|
||||
"filename",
|
||||
] as const;
|
||||
|
||||
const CONTENT_ALIASES = [
|
||||
"content",
|
||||
"content_preview",
|
||||
"snippet",
|
||||
"page_content",
|
||||
"body",
|
||||
"text",
|
||||
"summary",
|
||||
"abstract",
|
||||
"description",
|
||||
"desc",
|
||||
"excerpt",
|
||||
"preview",
|
||||
"passage",
|
||||
"paragraph",
|
||||
"chunk",
|
||||
"chunk_text",
|
||||
"document",
|
||||
"doc_content",
|
||||
"answer",
|
||||
"highlight",
|
||||
"context",
|
||||
"message",
|
||||
] as const;
|
||||
|
||||
const URL_ALIASES = ["url", "href", "link", "uri", "source_url", "web_url", "page_url"] as const;
|
||||
|
||||
const SOURCE_ALIASES = [
|
||||
"source",
|
||||
"source1",
|
||||
"source_name",
|
||||
"origin",
|
||||
"site",
|
||||
"site_name",
|
||||
"publisher",
|
||||
"domain",
|
||||
"from",
|
||||
"channel",
|
||||
"database",
|
||||
"knowledge_base",
|
||||
"kb",
|
||||
] as const;
|
||||
|
||||
const DATE_ALIASES = [
|
||||
"published_date",
|
||||
"publishedDate",
|
||||
"publish_date",
|
||||
"date",
|
||||
"m_publish",
|
||||
"published_at",
|
||||
"publish_time",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"time",
|
||||
] as const;
|
||||
|
||||
const TITLE_KEY_PATTERN =
|
||||
/(?:^|_)(?:title|name|heading|headline|subject)(?:$|_)/i;
|
||||
const CONTENT_KEY_PATTERN =
|
||||
/(?:^|_)(?:content|snippet|summary|abstract|body|text|preview|passage|excerpt|desc|description|chunk|paragraph|document|answer|highlight|context|message)(?:$|_)/i;
|
||||
const URL_KEY_PATTERN = /(?:^|_)(?:url|href|link|uri)(?:$|_)/i;
|
||||
const SOURCE_KEY_PATTERN =
|
||||
/(?:^|_)(?:source|origin|site|publisher|domain|from|channel|database|kb)(?:$|_)/i;
|
||||
const DATE_KEY_PATTERN =
|
||||
/(?:^|_)(?:date|time|published|publish|created|updated)(?:$|_)/i;
|
||||
|
||||
function pickString(...values: unknown[]): string | undefined {
|
||||
for (const value of values) {
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
return value.trim();
|
||||
}
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function looksLikeUrl(value: string): boolean {
|
||||
return /^https?:\/\//i.test(value) || /^www\./i.test(value);
|
||||
}
|
||||
|
||||
function collectNestedRecords(
|
||||
record: Record<string, unknown>,
|
||||
): Record<string, unknown>[] {
|
||||
const layers: Record<string, unknown>[] = [record];
|
||||
for (const key of NESTED_META_KEYS) {
|
||||
const nested = asRecord(record[key]);
|
||||
if (nested) {
|
||||
layers.push(nested);
|
||||
}
|
||||
}
|
||||
return layers;
|
||||
}
|
||||
|
||||
function pickFromLayers(
|
||||
layers: Record<string, unknown>[],
|
||||
aliases: readonly string[],
|
||||
): string | undefined {
|
||||
for (const layer of layers) {
|
||||
for (const alias of aliases) {
|
||||
const value = pickString(layer[alias]);
|
||||
if (value) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function pickByKeyPattern(
|
||||
layers: Record<string, unknown>[],
|
||||
pattern: RegExp,
|
||||
options?: { excludeUrl?: boolean; maxLength?: number },
|
||||
): string | undefined {
|
||||
for (const layer of layers) {
|
||||
for (const [key, value] of Object.entries(layer)) {
|
||||
if (SKIP_HEURISTIC_KEYS.has(key.toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
if (!pattern.test(key)) {
|
||||
continue;
|
||||
}
|
||||
const text = pickString(value);
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
if (options?.excludeUrl && looksLikeUrl(text)) {
|
||||
continue;
|
||||
}
|
||||
if (options?.maxLength !== undefined && text.length > options.maxLength) {
|
||||
continue;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function pickLongestContentFallback(
|
||||
layers: Record<string, unknown>[],
|
||||
reserved: Set<string>,
|
||||
): string | undefined {
|
||||
let best: string | undefined;
|
||||
for (const layer of layers) {
|
||||
for (const [key, value] of Object.entries(layer)) {
|
||||
const lower = key.toLowerCase();
|
||||
if (SKIP_HEURISTIC_KEYS.has(lower)) {
|
||||
continue;
|
||||
}
|
||||
if (TITLE_KEY_PATTERN.test(key) || URL_KEY_PATTERN.test(key)) {
|
||||
continue;
|
||||
}
|
||||
const text = pickString(value);
|
||||
if (!text || text.length < 8) {
|
||||
continue;
|
||||
}
|
||||
if (reserved.has(text)) {
|
||||
continue;
|
||||
}
|
||||
if (looksLikeUrl(text) && text.length < 120) {
|
||||
continue;
|
||||
}
|
||||
if (!best || text.length > best.length) {
|
||||
best = text;
|
||||
}
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function pickShortTitleFallback(
|
||||
layers: Record<string, unknown>[],
|
||||
reserved: Set<string>,
|
||||
): string | undefined {
|
||||
let best: string | undefined;
|
||||
for (const layer of layers) {
|
||||
for (const [key, value] of Object.entries(layer)) {
|
||||
if (URL_KEY_PATTERN.test(key) || CONTENT_KEY_PATTERN.test(key)) {
|
||||
continue;
|
||||
}
|
||||
const text = pickString(value);
|
||||
if (!text || text.length > 200 || text.length < 1) {
|
||||
continue;
|
||||
}
|
||||
if (reserved.has(text)) {
|
||||
continue;
|
||||
}
|
||||
if (looksLikeUrl(text)) {
|
||||
continue;
|
||||
}
|
||||
if (!best || text.length < best.length) {
|
||||
best = text;
|
||||
}
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function normalizeSearchResultItem(raw: unknown): SearchResultItem | null {
|
||||
const record = asRecord(raw);
|
||||
if (!record) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const layers = collectNestedRecords(record);
|
||||
|
||||
let title =
|
||||
pickFromLayers(layers, TITLE_ALIASES) ??
|
||||
pickByKeyPattern(layers, TITLE_KEY_PATTERN, { maxLength: 200 });
|
||||
let content =
|
||||
pickFromLayers(layers, CONTENT_ALIASES) ??
|
||||
pickByKeyPattern(layers, CONTENT_KEY_PATTERN, { excludeUrl: true });
|
||||
let url =
|
||||
pickFromLayers(layers, URL_ALIASES) ??
|
||||
pickByKeyPattern(layers, URL_KEY_PATTERN);
|
||||
let source =
|
||||
pickFromLayers(layers, SOURCE_ALIASES) ??
|
||||
pickByKeyPattern(layers, SOURCE_KEY_PATTERN, {
|
||||
excludeUrl: true,
|
||||
maxLength: 120,
|
||||
});
|
||||
const date =
|
||||
pickFromLayers(layers, DATE_ALIASES) ??
|
||||
pickByKeyPattern(layers, DATE_KEY_PATTERN, { maxLength: 64 });
|
||||
|
||||
const reserved = new Set<string>();
|
||||
for (const value of [title, content, url, source, date]) {
|
||||
if (value) {
|
||||
reserved.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
if (!content) {
|
||||
content = pickLongestContentFallback(layers, reserved);
|
||||
if (content) {
|
||||
reserved.add(content);
|
||||
}
|
||||
}
|
||||
if (!title) {
|
||||
title = pickShortTitleFallback(layers, reserved);
|
||||
}
|
||||
|
||||
if (!title && !content && !url) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
url,
|
||||
snippet: content,
|
||||
content,
|
||||
source,
|
||||
published_date: date,
|
||||
publishedDate: date,
|
||||
date,
|
||||
};
|
||||
}
|
||||
|
||||
function extractRawResults(payload: unknown, depth = 0): unknown[] | null {
|
||||
if (depth > 4) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (Array.isArray(payload)) {
|
||||
return payload.length > 0 ? payload : null;
|
||||
}
|
||||
|
||||
const record = asRecord(payload);
|
||||
if (!record) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (const key of RESULT_LIST_KEYS) {
|
||||
const nested = record[key];
|
||||
if (Array.isArray(nested) && nested.length > 0) {
|
||||
return nested;
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of WRAPPER_KEYS) {
|
||||
const nested = record[key];
|
||||
const found = extractRawResults(nested, depth + 1);
|
||||
if (found) {
|
||||
return found;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseResultPayload(
|
||||
result: string | Record<string, unknown> | undefined,
|
||||
): unknown {
|
||||
if (result === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof result === "string") {
|
||||
const trimmed = result.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(trimmed) as unknown;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** True when the tool return value looks like a retrieval hit list. */
|
||||
export function hasSearchResultPayload(
|
||||
result: string | Record<string, unknown> | undefined,
|
||||
): boolean {
|
||||
return normalizeSearchResults(result).length > 0;
|
||||
}
|
||||
|
||||
export function normalizeSearchResults(
|
||||
result: string | Record<string, unknown> | undefined,
|
||||
): SearchResultItem[] {
|
||||
const payload = parseResultPayload(result);
|
||||
const rawList = extractRawResults(payload);
|
||||
if (!rawList) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const items: SearchResultItem[] = [];
|
||||
for (const raw of rawList) {
|
||||
const item = normalizeSearchResultItem(raw);
|
||||
if (item) {
|
||||
items.push(item);
|
||||
}
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
export function resolveSearchQuery(
|
||||
args: Record<string, unknown>,
|
||||
result: string | Record<string, unknown> | undefined,
|
||||
): string | undefined {
|
||||
if (typeof args.query === "string" && args.query.trim()) {
|
||||
return args.query.trim();
|
||||
}
|
||||
const payload = asRecord(parseResultPayload(result));
|
||||
if (!payload) {
|
||||
return undefined;
|
||||
}
|
||||
const queryKeys = ["query", "q", "keyword", "keywords", "search_query"] as const;
|
||||
for (const key of queryKeys) {
|
||||
const value = payload[key];
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
Reference in new issue
Block a user