大模型速度增加提示
This commit is contained in:
1 parent
4e764350a4
commit
3ea33e1dee
52 files changed
+4673
-176
No files matched your search
@@ -0,0 +1,235 @@
|
||||
"use client";
|
||||
|
||||
import type { Message } from "@langchain/langgraph-sdk";
|
||||
import { createContext, useContext, useEffect, useRef, useState } from "react";
|
||||
|
||||
import { extractReasoningContentFromMessage } from "@/core/messages/utils";
|
||||
import { getUsageMetadata } from "@/core/messages/usage";
|
||||
import { textOfMessage } from "@/core/threads/utils";
|
||||
|
||||
/**
|
||||
* Two characters ≈ one token. Same heuristic we use elsewhere; gives a stable
|
||||
* speed indicator for mixed Chinese/English replies without dragging in a
|
||||
* real tokeniser.
|
||||
*/
|
||||
const CHARS_PER_TOKEN = 2;
|
||||
|
||||
/** Sample throttle while a message is streaming. */
|
||||
const SAMPLE_THROTTLE_MS = 60;
|
||||
|
||||
/**
|
||||
* Per-AI-message metric: real-time during streaming, frozen once the run ends.
|
||||
*/
|
||||
export interface MessageMetric {
|
||||
/** Rolling tokens/sec, or 0 when we cannot estimate yet. */
|
||||
tokensPerSec: number;
|
||||
/**
|
||||
* Wall-clock duration from when the message first appeared with text to
|
||||
* either *now* (active) or to when the stream ended (frozen).
|
||||
*/
|
||||
durationMs: number;
|
||||
/** True while this is the currently streaming message. */
|
||||
isActive: boolean;
|
||||
/**
|
||||
* Whether the speed value was computed from real usage_metadata output
|
||||
* tokens (``true``) vs. a char-count fallback (``false``). UI uses this
|
||||
* only to add a "~" prefix when the value is approximate.
|
||||
*/
|
||||
fromUsage: boolean;
|
||||
}
|
||||
|
||||
type Sample = { at: number; chars: number };
|
||||
|
||||
type TimingEntry = {
|
||||
firstSeenAt: number;
|
||||
endedAt: number | null;
|
||||
samples: Sample[];
|
||||
lastChars: number;
|
||||
/** True iff we *have already* observed `usage_metadata.output_tokens`. */
|
||||
hadUsage: boolean;
|
||||
};
|
||||
|
||||
function isAiMessage(message: Message): boolean {
|
||||
return message.type === "ai";
|
||||
}
|
||||
|
||||
function getMessageId(message: Message, fallbackIndex: number): string {
|
||||
const id = (message.id as string | undefined) ?? undefined;
|
||||
return id ?? `idx-${fallbackIndex}`;
|
||||
}
|
||||
|
||||
function messageText(message: Message): string {
|
||||
// Count both the visible answer text *and* the reasoning trace so the speed
|
||||
// indicator stays meaningful during the thinking phase — for reasoning
|
||||
// models the answer body can be empty for tens of seconds while the model
|
||||
// streams reasoning tokens, and ignoring those would make the meter sit at
|
||||
// zero. We treat the two as one continuous stream of generated characters.
|
||||
const body = textOfMessage(message);
|
||||
const reasoning = extractReasoningContentFromMessage(message) ?? "";
|
||||
const bodyText = typeof body === "string" ? body : "";
|
||||
return bodyText + reasoning;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a Map keyed by AI message id → live metric snapshot.
|
||||
*
|
||||
* Recording strategy:
|
||||
* - Whenever a message renders with text, we anchor `firstSeenAt`.
|
||||
* - On every text-length change we push a sample and recompute the rate.
|
||||
* - When `isLoading` flips to false we mark *all* still-active messages as
|
||||
* ended so subsequent renders return a frozen duration.
|
||||
* - Token totals come from usage_metadata when present (most accurate);
|
||||
* otherwise we fall back to char-count / 2.
|
||||
*/
|
||||
export function useMessageMetrics(
|
||||
messages: Message[],
|
||||
isLoading: boolean,
|
||||
): Map<string, MessageMetric> {
|
||||
const timingRef = useRef<Map<string, TimingEntry>>(new Map());
|
||||
const lastSampleAtRef = useRef(0);
|
||||
// Bump a counter to force recomputation when samples push but message refs
|
||||
// didn't change — needed because the timing map mutation alone wouldn't
|
||||
// re-render anything.
|
||||
const [, setTick] = useState(0);
|
||||
const lastActiveIdRef = useRef<string | null>(null);
|
||||
|
||||
// Walk messages and update timings. We intentionally read/write the ref
|
||||
// synchronously inside an effect so React isn't tracking the map identity.
|
||||
useEffect(() => {
|
||||
const now = performance.now();
|
||||
let activeId: string | null = null;
|
||||
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const message = messages[i];
|
||||
if (!message || !isAiMessage(message)) continue;
|
||||
const id = getMessageId(message, i);
|
||||
const text = messageText(message);
|
||||
if (!text) continue;
|
||||
|
||||
let entry = timingRef.current.get(id);
|
||||
if (!entry) {
|
||||
entry = {
|
||||
firstSeenAt: now,
|
||||
endedAt: null,
|
||||
samples: [{ at: now, chars: text.length }],
|
||||
lastChars: text.length,
|
||||
hadUsage: false,
|
||||
};
|
||||
timingRef.current.set(id, entry);
|
||||
}
|
||||
|
||||
// Track the most recent AI message as the active one (while streaming).
|
||||
activeId = id;
|
||||
|
||||
// Sample only when actively streaming OR when content actually grew —
|
||||
// throttle to keep arrays bounded.
|
||||
const grew = text.length !== entry.lastChars;
|
||||
if (grew && now - lastSampleAtRef.current >= SAMPLE_THROTTLE_MS) {
|
||||
entry.samples.push({ at: now, chars: text.length });
|
||||
entry.lastChars = text.length;
|
||||
if (entry.samples.length > 256) {
|
||||
entry.samples.splice(0, entry.samples.length - 256);
|
||||
}
|
||||
lastSampleAtRef.current = now;
|
||||
} else if (grew) {
|
||||
// Still update lastChars so the next render captures it eventually.
|
||||
entry.lastChars = text.length;
|
||||
}
|
||||
|
||||
if (getUsageMetadata(message)) {
|
||||
entry.hadUsage = true;
|
||||
}
|
||||
}
|
||||
|
||||
// When streaming stops, freeze any non-ended entry to the current time.
|
||||
if (!isLoading) {
|
||||
for (const entry of timingRef.current.values()) {
|
||||
if (entry.endedAt === null) {
|
||||
entry.endedAt = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lastActiveIdRef.current = activeId;
|
||||
setTick((v) => v + 1);
|
||||
}, [messages, isLoading]);
|
||||
|
||||
// Drive a slow tick while streaming so live durations refresh smoothly.
|
||||
useEffect(() => {
|
||||
if (!isLoading) return;
|
||||
const interval = window.setInterval(() => setTick((v) => v + 1), 300);
|
||||
return () => window.clearInterval(interval);
|
||||
}, [isLoading]);
|
||||
|
||||
// Build the snapshot map. We compute on every render so consumers see the
|
||||
// freshest duration; the underlying timing storage is the source of truth.
|
||||
const out = new Map<string, MessageMetric>();
|
||||
const now = performance.now();
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const message = messages[i];
|
||||
if (!message || !isAiMessage(message)) continue;
|
||||
const id = getMessageId(message, i);
|
||||
const entry = timingRef.current.get(id);
|
||||
if (!entry) continue;
|
||||
|
||||
const endedAt = entry.endedAt ?? now;
|
||||
const durationMs = Math.max(0, endedAt - entry.firstSeenAt);
|
||||
const isActive = isLoading && id === lastActiveIdRef.current;
|
||||
|
||||
let tokensPerSec = 0;
|
||||
let fromUsage = false;
|
||||
const usage = getUsageMetadata(message);
|
||||
if (usage && usage.outputTokens > 0 && durationMs > 0) {
|
||||
tokensPerSec = (usage.outputTokens / durationMs) * 1000;
|
||||
fromUsage = true;
|
||||
} else {
|
||||
// Char-count fallback. Uses the rolling window during streaming so a
|
||||
// mid-stream snapshot reflects current speed rather than average since
|
||||
// the start.
|
||||
if (isActive && entry.samples.length >= 2) {
|
||||
const samples = entry.samples;
|
||||
let start = samples.findIndex((s) => s.at >= now - 1500);
|
||||
if (start === -1) start = samples.length - 2;
|
||||
if (start < 0) start = 0;
|
||||
const head = samples[start];
|
||||
const tail = samples[samples.length - 1];
|
||||
if (head && tail) {
|
||||
const elapsedMs = tail.at - head.at;
|
||||
const chars = Math.max(0, tail.chars - head.chars);
|
||||
if (elapsedMs > 0) {
|
||||
tokensPerSec = (chars / elapsedMs) * 1000 / CHARS_PER_TOKEN;
|
||||
}
|
||||
}
|
||||
} else if (durationMs > 0) {
|
||||
// Finished message without usage data: average over full duration.
|
||||
const chars = entry.lastChars;
|
||||
tokensPerSec = (chars / durationMs) * 1000 / CHARS_PER_TOKEN;
|
||||
}
|
||||
}
|
||||
|
||||
out.set(id, {
|
||||
tokensPerSec,
|
||||
durationMs,
|
||||
isActive,
|
||||
fromUsage,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
const EMPTY_METRICS: Map<string, MessageMetric> = new Map();
|
||||
|
||||
/**
|
||||
* Context for sharing the metrics map computed by ``useMessageMetrics`` from
|
||||
* MessageList down to per-message renderers like MessageListItem, without
|
||||
* threading a prop through every grouping branch.
|
||||
*/
|
||||
export const MessageMetricsContext = createContext<Map<string, MessageMetric>>(
|
||||
EMPTY_METRICS,
|
||||
);
|
||||
|
||||
export function useMessageMetric(messageId: string | undefined): MessageMetric | undefined {
|
||||
const metrics = useContext(MessageMetricsContext);
|
||||
return messageId ? metrics.get(messageId) : undefined;
|
||||
}
|
||||
Reference in new issue
Block a user