263 lines
9.3 KiB
TypeScript
263 lines
9.3 KiB
TypeScript
"use client";
|
|
|
|
import type { Message } from "@langchain/langgraph-sdk";
|
|
import { createContext, useContext, useEffect, useRef, useState } from "react";
|
|
|
|
import { extractReasoningContentFromMessage } from "@/core/messages/utils";
|
|
import { getUsageMetadata } from "@/core/messages/usage";
|
|
import { textOfMessage } from "@/core/threads/utils";
|
|
|
|
/**
|
|
* Two characters ≈ one token. Same heuristic we use elsewhere; gives a stable
|
|
* speed indicator for mixed Chinese/English replies without dragging in a
|
|
* real tokeniser.
|
|
*/
|
|
const CHARS_PER_TOKEN = 2;
|
|
|
|
/** Sample throttle while a message is streaming. */
|
|
const SAMPLE_THROTTLE_MS = 60;
|
|
|
|
/**
|
|
* Per-AI-message metric: real-time during streaming, frozen once the run ends.
|
|
*/
|
|
export interface MessageMetric {
|
|
/** Rolling tokens/sec, or 0 when we cannot estimate yet. */
|
|
tokensPerSec: number;
|
|
/**
|
|
* Wall-clock duration from when the message first appeared with text to
|
|
* either *now* (active) or to when the stream ended (frozen).
|
|
*/
|
|
durationMs: number;
|
|
/** True while this is the currently streaming message. */
|
|
isActive: boolean;
|
|
/**
|
|
* Whether the speed value was computed from real usage_metadata output
|
|
* tokens (``true``) vs. a char-count fallback (``false``). UI uses this
|
|
* only to add a "~" prefix when the value is approximate.
|
|
*/
|
|
fromUsage: boolean;
|
|
/**
|
|
* True iff this is the turn-final AI message — the last AI message before
|
|
* the next human message. An agentic turn makes several LLM calls, each its
|
|
* own AI message; the speed pill is shown only on the turn-final one so a
|
|
* turn renders a single pill instead of one per call.
|
|
*/
|
|
isTurnFinal: boolean;
|
|
}
|
|
|
|
type Sample = { at: number; chars: number };
|
|
|
|
type TimingEntry = {
|
|
firstSeenAt: number;
|
|
endedAt: number | null;
|
|
samples: Sample[];
|
|
lastChars: number;
|
|
/** True iff we *have already* observed `usage_metadata.output_tokens`. */
|
|
hadUsage: boolean;
|
|
};
|
|
|
|
function isAiMessage(message: Message): boolean {
|
|
return message.type === "ai";
|
|
}
|
|
|
|
function getMessageId(message: Message, fallbackIndex: number): string {
|
|
const id = (message.id as string | undefined) ?? undefined;
|
|
return id ?? `idx-${fallbackIndex}`;
|
|
}
|
|
|
|
function messageText(message: Message): string {
|
|
// Count both the visible answer text *and* the reasoning trace so the speed
|
|
// indicator stays meaningful during the thinking phase — for reasoning
|
|
// models the answer body can be empty for tens of seconds while the model
|
|
// streams reasoning tokens, and ignoring those would make the meter sit at
|
|
// zero. We treat the two as one continuous stream of generated characters.
|
|
const body = textOfMessage(message);
|
|
const reasoning = extractReasoningContentFromMessage(message) ?? "";
|
|
const bodyText = typeof body === "string" ? body : "";
|
|
return bodyText + reasoning;
|
|
}
|
|
|
|
/**
|
|
* Returns a Map keyed by AI message id → live metric snapshot.
|
|
*
|
|
* Recording strategy:
|
|
* - Whenever a message renders with text, we anchor `firstSeenAt`.
|
|
* - On every text-length change we push a sample and recompute the rate.
|
|
* - When `isLoading` flips to false we mark *all* still-active messages as
|
|
* ended so subsequent renders return a frozen duration.
|
|
* - Token totals come from usage_metadata when present (most accurate);
|
|
* otherwise we fall back to char-count / 2.
|
|
*/
|
|
export function useMessageMetrics(
|
|
messages: Message[],
|
|
isLoading: boolean,
|
|
): Map<string, MessageMetric> {
|
|
const timingRef = useRef<Map<string, TimingEntry>>(new Map());
|
|
const lastSampleAtRef = useRef(0);
|
|
// Bump a counter to force recomputation when samples push but message refs
|
|
// didn't change — needed because the timing map mutation alone wouldn't
|
|
// re-render anything.
|
|
const [, setTick] = useState(0);
|
|
const lastActiveIdRef = useRef<string | null>(null);
|
|
|
|
// Walk messages and update timings. We intentionally read/write the ref
|
|
// synchronously inside an effect so React isn't tracking the map identity.
|
|
useEffect(() => {
|
|
const now = performance.now();
|
|
let activeId: string | null = null;
|
|
|
|
for (let i = 0; i < messages.length; i++) {
|
|
const message = messages[i];
|
|
if (!message || !isAiMessage(message)) continue;
|
|
const id = getMessageId(message, i);
|
|
const text = messageText(message);
|
|
if (!text) continue;
|
|
|
|
let entry = timingRef.current.get(id);
|
|
if (!entry) {
|
|
entry = {
|
|
firstSeenAt: now,
|
|
endedAt: null,
|
|
samples: [{ at: now, chars: text.length }],
|
|
lastChars: text.length,
|
|
hadUsage: false,
|
|
};
|
|
timingRef.current.set(id, entry);
|
|
}
|
|
|
|
// Track the most recent AI message as the active one (while streaming).
|
|
activeId = id;
|
|
|
|
// Sample only when actively streaming OR when content actually grew —
|
|
// throttle to keep arrays bounded.
|
|
const grew = text.length !== entry.lastChars;
|
|
if (grew && now - lastSampleAtRef.current >= SAMPLE_THROTTLE_MS) {
|
|
entry.samples.push({ at: now, chars: text.length });
|
|
entry.lastChars = text.length;
|
|
if (entry.samples.length > 256) {
|
|
entry.samples.splice(0, entry.samples.length - 256);
|
|
}
|
|
lastSampleAtRef.current = now;
|
|
} else if (grew) {
|
|
// Still update lastChars so the next render captures it eventually.
|
|
entry.lastChars = text.length;
|
|
}
|
|
|
|
if (getUsageMetadata(message)) {
|
|
entry.hadUsage = true;
|
|
}
|
|
}
|
|
|
|
// When streaming stops, freeze any non-ended entry to the current time.
|
|
if (!isLoading) {
|
|
for (const entry of timingRef.current.values()) {
|
|
if (entry.endedAt === null) {
|
|
entry.endedAt = now;
|
|
}
|
|
}
|
|
}
|
|
|
|
lastActiveIdRef.current = activeId;
|
|
setTick((v) => v + 1);
|
|
}, [messages, isLoading]);
|
|
|
|
// Drive a slow tick while streaming so live durations refresh smoothly.
|
|
useEffect(() => {
|
|
if (!isLoading) return;
|
|
const interval = window.setInterval(() => setTick((v) => v + 1), 300);
|
|
return () => window.clearInterval(interval);
|
|
}, [isLoading]);
|
|
|
|
// Identify the turn-final AI message of each turn — the last AI message
|
|
// (with a recorded metric) before the next human message. Only these show
|
|
// the speed pill, collapsing a multi-LLM-call turn to a single pill.
|
|
const turnFinalIds = new Set<string>();
|
|
let pendingTurnFinal: string | null = null;
|
|
for (let i = 0; i < messages.length; i++) {
|
|
const message = messages[i];
|
|
if (!message) continue;
|
|
if (message.type === "human") {
|
|
if (pendingTurnFinal) turnFinalIds.add(pendingTurnFinal);
|
|
pendingTurnFinal = null;
|
|
continue;
|
|
}
|
|
if (isAiMessage(message) && timingRef.current.has(getMessageId(message, i))) {
|
|
pendingTurnFinal = getMessageId(message, i);
|
|
}
|
|
}
|
|
if (pendingTurnFinal) turnFinalIds.add(pendingTurnFinal);
|
|
|
|
// Build the snapshot map. We compute on every render so consumers see the
|
|
// freshest duration; the underlying timing storage is the source of truth.
|
|
const out = new Map<string, MessageMetric>();
|
|
const now = performance.now();
|
|
for (let i = 0; i < messages.length; i++) {
|
|
const message = messages[i];
|
|
if (!message || !isAiMessage(message)) continue;
|
|
const id = getMessageId(message, i);
|
|
const entry = timingRef.current.get(id);
|
|
if (!entry) continue;
|
|
|
|
const endedAt = entry.endedAt ?? now;
|
|
const durationMs = Math.max(0, endedAt - entry.firstSeenAt);
|
|
const isActive = isLoading && id === lastActiveIdRef.current;
|
|
|
|
let tokensPerSec = 0;
|
|
let fromUsage = false;
|
|
const usage = getUsageMetadata(message);
|
|
if (usage && usage.outputTokens > 0 && durationMs > 0) {
|
|
tokensPerSec = (usage.outputTokens / durationMs) * 1000;
|
|
fromUsage = true;
|
|
} else {
|
|
// Char-count fallback. Uses the rolling window during streaming so a
|
|
// mid-stream snapshot reflects current speed rather than average since
|
|
// the start.
|
|
if (isActive && entry.samples.length >= 2) {
|
|
const samples = entry.samples;
|
|
let start = samples.findIndex((s) => s.at >= now - 1500);
|
|
if (start === -1) start = samples.length - 2;
|
|
if (start < 0) start = 0;
|
|
const head = samples[start];
|
|
const tail = samples[samples.length - 1];
|
|
if (head && tail) {
|
|
const elapsedMs = tail.at - head.at;
|
|
const chars = Math.max(0, tail.chars - head.chars);
|
|
if (elapsedMs > 0) {
|
|
tokensPerSec = (chars / elapsedMs) * 1000 / CHARS_PER_TOKEN;
|
|
}
|
|
}
|
|
} else if (durationMs > 0) {
|
|
// Finished message without usage data: average over full duration.
|
|
const chars = entry.lastChars;
|
|
tokensPerSec = (chars / durationMs) * 1000 / CHARS_PER_TOKEN;
|
|
}
|
|
}
|
|
|
|
out.set(id, {
|
|
tokensPerSec,
|
|
durationMs,
|
|
isActive,
|
|
fromUsage,
|
|
isTurnFinal: turnFinalIds.has(id),
|
|
});
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
const EMPTY_METRICS: Map<string, MessageMetric> = new Map();
|
|
|
|
/**
|
|
* Context for sharing the metrics map computed by ``useMessageMetrics`` from
|
|
* MessageList down to per-message renderers like MessageListItem, without
|
|
* threading a prop through every grouping branch.
|
|
*/
|
|
export const MessageMetricsContext = createContext<Map<string, MessageMetric>>(
|
|
EMPTY_METRICS,
|
|
);
|
|
|
|
export function useMessageMetric(messageId: string | undefined): MessageMetric | undefined {
|
|
const metrics = useContext(MessageMetricsContext);
|
|
return messageId ? metrics.get(messageId) : undefined;
|
|
}
|