大模型速度增加提示

This commit is contained in:
吉鑫升 committed 2026-05-14 19:12:14 +08:00
1 parent 4e764350a4
commit 3ea33e1dee
52 files changed
+4673 -176

No files matched your search

@@ -0,0 +1,235 @@
"use client";
import type { Message } from "@langchain/langgraph-sdk";
import { createContext, useContext, useEffect, useRef, useState } from "react";
import { extractReasoningContentFromMessage } from "@/core/messages/utils";
import { getUsageMetadata } from "@/core/messages/usage";
import { textOfMessage } from "@/core/threads/utils";
/**
* Two characters ≈ one token. Same heuristic we use elsewhere; gives a stable
* speed indicator for mixed Chinese/English replies without dragging in a
* real tokeniser.
*/
const CHARS_PER_TOKEN = 2;
/** Sample throttle while a message is streaming. */
const SAMPLE_THROTTLE_MS = 60;
/**
* Per-AI-message metric: real-time during streaming, frozen once the run ends.
*/
export interface MessageMetric {
/** Rolling tokens/sec, or 0 when we cannot estimate yet. */
tokensPerSec: number;
/**
* Wall-clock duration from when the message first appeared with text to
* either *now* (active) or to when the stream ended (frozen).
*/
durationMs: number;
/** True while this is the currently streaming message. */
isActive: boolean;
/**
* Whether the speed value was computed from real usage_metadata output
* tokens (``true``) vs. a char-count fallback (``false``). UI uses this
* only to add a "~" prefix when the value is approximate.
*/
fromUsage: boolean;
}
type Sample = { at: number; chars: number };
type TimingEntry = {
firstSeenAt: number;
endedAt: number | null;
samples: Sample[];
lastChars: number;
/** True iff we *have already* observed `usage_metadata.output_tokens`. */
hadUsage: boolean;
};
function isAiMessage(message: Message): boolean {
return message.type === "ai";
}
function getMessageId(message: Message, fallbackIndex: number): string {
const id = (message.id as string | undefined) ?? undefined;
return id ?? `idx-${fallbackIndex}`;
}
function messageText(message: Message): string {
// Count both the visible answer text *and* the reasoning trace so the speed
// indicator stays meaningful during the thinking phase — for reasoning
// models the answer body can be empty for tens of seconds while the model
// streams reasoning tokens, and ignoring those would make the meter sit at
// zero. We treat the two as one continuous stream of generated characters.
const body = textOfMessage(message);
const reasoning = extractReasoningContentFromMessage(message) ?? "";
const bodyText = typeof body === "string" ? body : "";
return bodyText + reasoning;
}
/**
* Returns a Map keyed by AI message id → live metric snapshot.
*
* Recording strategy:
* - Whenever a message renders with text, we anchor `firstSeenAt`.
* - On every text-length change we push a sample and recompute the rate.
* - When `isLoading` flips to false we mark *all* still-active messages as
* ended so subsequent renders return a frozen duration.
* - Token totals come from usage_metadata when present (most accurate);
* otherwise we fall back to char-count / 2.
*/
export function useMessageMetrics(
messages: Message[],
isLoading: boolean,
): Map<string, MessageMetric> {
const timingRef = useRef<Map<string, TimingEntry>>(new Map());
const lastSampleAtRef = useRef(0);
// Bump a counter to force recomputation when samples push but message refs
// didn't change — needed because the timing map mutation alone wouldn't
// re-render anything.
const [, setTick] = useState(0);
const lastActiveIdRef = useRef<string | null>(null);
// Walk messages and update timings. We intentionally read/write the ref
// synchronously inside an effect so React isn't tracking the map identity.
useEffect(() => {
const now = performance.now();
let activeId: string | null = null;
for (let i = 0; i < messages.length; i++) {
const message = messages[i];
if (!message || !isAiMessage(message)) continue;
const id = getMessageId(message, i);
const text = messageText(message);
if (!text) continue;
let entry = timingRef.current.get(id);
if (!entry) {
entry = {
firstSeenAt: now,
endedAt: null,
samples: [{ at: now, chars: text.length }],
lastChars: text.length,
hadUsage: false,
};
timingRef.current.set(id, entry);
}
// Track the most recent AI message as the active one (while streaming).
activeId = id;
// Sample only when actively streaming OR when content actually grew —
// throttle to keep arrays bounded.
const grew = text.length !== entry.lastChars;
if (grew && now - lastSampleAtRef.current >= SAMPLE_THROTTLE_MS) {
entry.samples.push({ at: now, chars: text.length });
entry.lastChars = text.length;
if (entry.samples.length > 256) {
entry.samples.splice(0, entry.samples.length - 256);
}
lastSampleAtRef.current = now;
} else if (grew) {
// Still update lastChars so the next render captures it eventually.
entry.lastChars = text.length;
}
if (getUsageMetadata(message)) {
entry.hadUsage = true;
}
}
// When streaming stops, freeze any non-ended entry to the current time.
if (!isLoading) {
for (const entry of timingRef.current.values()) {
if (entry.endedAt === null) {
entry.endedAt = now;
}
}
}
lastActiveIdRef.current = activeId;
setTick((v) => v + 1);
}, [messages, isLoading]);
// Drive a slow tick while streaming so live durations refresh smoothly.
useEffect(() => {
if (!isLoading) return;
const interval = window.setInterval(() => setTick((v) => v + 1), 300);
return () => window.clearInterval(interval);
}, [isLoading]);
// Build the snapshot map. We compute on every render so consumers see the
// freshest duration; the underlying timing storage is the source of truth.
const out = new Map<string, MessageMetric>();
const now = performance.now();
for (let i = 0; i < messages.length; i++) {
const message = messages[i];
if (!message || !isAiMessage(message)) continue;
const id = getMessageId(message, i);
const entry = timingRef.current.get(id);
if (!entry) continue;
const endedAt = entry.endedAt ?? now;
const durationMs = Math.max(0, endedAt - entry.firstSeenAt);
const isActive = isLoading && id === lastActiveIdRef.current;
let tokensPerSec = 0;
let fromUsage = false;
const usage = getUsageMetadata(message);
if (usage && usage.outputTokens > 0 && durationMs > 0) {
tokensPerSec = (usage.outputTokens / durationMs) * 1000;
fromUsage = true;
} else {
// Char-count fallback. Uses the rolling window during streaming so a
// mid-stream snapshot reflects current speed rather than average since
// the start.
if (isActive && entry.samples.length >= 2) {
const samples = entry.samples;
let start = samples.findIndex((s) => s.at >= now - 1500);
if (start === -1) start = samples.length - 2;
if (start < 0) start = 0;
const head = samples[start];
const tail = samples[samples.length - 1];
if (head && tail) {
const elapsedMs = tail.at - head.at;
const chars = Math.max(0, tail.chars - head.chars);
if (elapsedMs > 0) {
tokensPerSec = (chars / elapsedMs) * 1000 / CHARS_PER_TOKEN;
}
}
} else if (durationMs > 0) {
// Finished message without usage data: average over full duration.
const chars = entry.lastChars;
tokensPerSec = (chars / durationMs) * 1000 / CHARS_PER_TOKEN;
}
}
out.set(id, {
tokensPerSec,
durationMs,
isActive,
fromUsage,
});
}
return out;
}
const EMPTY_METRICS: Map<string, MessageMetric> = new Map();
/**
* Context for sharing the metrics map computed by ``useMessageMetrics`` from
* MessageList down to per-message renderers like MessageListItem, without
* threading a prop through every grouping branch.
*/
export const MessageMetricsContext = createContext<Map<string, MessageMetric>>(
EMPTY_METRICS,
);
export function useMessageMetric(messageId: string | undefined): MessageMetric | undefined {
const metrics = useContext(MessageMetricsContext);
return messageId ? metrics.get(messageId) : undefined;
}