sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -1,38 +1,93 @@
"""Feedback endpoints — create, list, stats, delete.
Allows users to submit thumbs-up/down feedback on runs,
optionally scoped to a specific message.
"""
"""Assistant-answer feedback and transparent user-feedback scoring APIs."""
from __future__ import annotations
import logging
from typing import Any
from enum import StrEnum
from typing import Any, Literal
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field, model_validator
from app.gateway.authz import require_permission
from app.gateway.deps import get_current_user, get_feedback_repo, get_run_store
from app.gateway.deps import (
get_current_user,
get_feedback_repo,
get_optional_user_from_request,
get_run_event_store,
get_run_store,
get_thread_store,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads", tags=["feedback"])
agent_score_router = APIRouter(prefix="/api/agents", tags=["feedback"])
_MESSAGE_SCAN_BATCH = 200
_MESSAGE_SCAN_MAX = 10_000
# ---------------------------------------------------------------------------
# Request / response models
# ---------------------------------------------------------------------------
async def _require_admin(request: Request) -> None:
"""Apply the project's admin role rule, preserving auth-disabled mode."""
user = await get_optional_user_from_request(request)
if user is None:
return
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="Agent feedback score is admin-only")
class FeedbackCreateRequest(BaseModel):
rating: int = Field(..., description="Feedback rating: +1 (positive) or -1 (negative)")
comment: str | None = Field(default=None, description="Optional text feedback")
message_id: str | None = Field(default=None, description="Optional: scope feedback to a specific message")
class FeedbackReason(StrEnum):
INCORRECT_OR_INCOMPLETE = "incorrect_or_incomplete"
UNMET_EXPECTATIONS = "unmet_expectations"
SLOW_OR_PROBLEMATIC = "slow_or_problematic"
STYLE_OR_TONE = "style_or_tone"
SAFETY_OR_LEGAL = "safety_or_legal"
OTHER = "other"
class FeedbackUpsertRequest(BaseModel):
rating: int = Field(..., description="Feedback rating: +1 (positive) or -1 (negative)")
comment: str | None = Field(default=None, description="Optional text feedback")
class FeedbackWriteRequest(BaseModel):
rating: Literal[-1, 1] = Field(
...,
description="+1 (excellent answer) or -1 (poor answer)",
)
reasons: list[FeedbackReason] = Field(default_factory=list, max_length=6)
detail: str | None = Field(default=None, max_length=2000)
# Kept for existing API clients. New clients should use ``detail``.
comment: str | None = Field(
default=None,
max_length=2000,
description="Deprecated compatibility alias for detail",
)
# Used only by the legacy run-scoped routes. Message-scoped routes take
# this value from the URL and reject a forged run/message association.
message_id: str | None = Field(default=None, min_length=1, max_length=64)
@model_validator(mode="after")
def validate_feedback_shape(self) -> FeedbackWriteRequest:
self.reasons = list(dict.fromkeys(self.reasons))
if self.rating == 1 and self.reasons:
raise ValueError("positive feedback must not include problem reasons")
if self.rating == -1 and not self.reasons:
raise ValueError("negative feedback requires at least one problem reason")
if self.detail is not None and self.comment is not None and self.detail != self.comment:
raise ValueError("detail and deprecated comment must match when both are provided")
return self
def normalized_detail(self) -> str | None:
value = self.detail if self.detail is not None else self.comment
if value is None:
return None
stripped = value.strip()
return stripped or None
def reason_values(self) -> list[str]:
return [reason.value for reason in self.reasons]
# Compatibility names used by OpenAPI clients generated from the prior API.
FeedbackCreateRequest = FeedbackWriteRequest
FeedbackUpsertRequest = FeedbackWriteRequest
class FeedbackResponse(BaseModel):
@@ -41,9 +96,13 @@ class FeedbackResponse(BaseModel):
thread_id: str
user_id: str | None = None
message_id: str | None = None
rating: int
agent_id: str
rating: Literal[-1, 1]
reasons: list[FeedbackReason] = Field(default_factory=list)
detail: str | None = None
comment: str | None = None
created_at: str = ""
updated_at: str = ""
class FeedbackStatsResponse(BaseModel):
@@ -53,9 +112,434 @@ class FeedbackStatsResponse(BaseModel):
negative: int = 0
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
class FeedbackContextResponse(BaseModel):
thread_id: str
message_id: str
run_id: str
agent_id: str
feedback: FeedbackResponse | None = None
class FeedbackContextsRequest(BaseModel):
message_ids: list[str] = Field(..., min_length=1, max_length=200)
@model_validator(mode="after")
def normalize_message_ids(self) -> FeedbackContextsRequest:
normalized: list[str] = []
for item in self.message_ids:
value = str(item).strip()
if not value or len(value) > 64:
raise ValueError("message_ids must contain non-empty ids up to 64 characters")
if value not in normalized:
normalized.append(value)
self.message_ids = normalized
return self
class FeedbackContextsResponse(BaseModel):
contexts: list[FeedbackContextResponse]
unresolved_message_ids: list[str]
class ScoreBandResponse(BaseModel):
key: str
label: str
min: float
max: float
class DimensionScoreResponse(BaseModel):
weight: float
score: float | None
evidence_count: int
positive_count: int
negative_count: int
mapped_reasons: list[FeedbackReason]
class ReasonBreakdownResponse(BaseModel):
count: int
rate: float
class AgentFeedbackScoreResponse(BaseModel):
agent_id: str
score: float | None
score_band: ScoreBandResponse
confidence: float = Field(..., ge=0, le=1)
confidence_level: Literal["none", "low", "medium", "high"]
sample_count: int
positive_count: int
negative_count: int
task_success_rate: float | None
task_score: float | None
dimensions: dict[str, DimensionScoreResponse]
reason_breakdown: dict[str, ReasonBreakdownResponse]
prior: dict[str, int | float]
formula: dict[str, str]
def _event_message_id(event: dict[str, Any]) -> str | None:
content = event.get("content")
if not isinstance(content, dict):
return None
value = content.get("id")
return str(value) if value else None
async def _find_ai_message_event(
request: Request,
*,
thread_id: str,
message_id: str,
) -> dict[str, Any]:
"""Resolve a checkpoint message id to its exact durable run event."""
found = await _find_ai_message_events(
request,
thread_id=thread_id,
message_ids={message_id},
)
if message_id in found:
return found[message_id]
raise HTTPException(
status_code=404,
detail=f"Assistant message {message_id} was not found in thread {thread_id}",
)
async def _find_ai_message_events(
request: Request,
*,
thread_id: str,
message_ids: set[str],
) -> dict[str, dict[str, Any]]:
"""Resolve up to 200 message ids in one bounded reverse event scan."""
event_store = get_run_event_store(request)
before_seq: int | None = None
scanned = 0
found: dict[str, dict[str, Any]] = {}
while scanned < _MESSAGE_SCAN_MAX and len(found) < len(message_ids):
rows = await event_store.list_messages(
thread_id,
limit=_MESSAGE_SCAN_BATCH,
before_seq=before_seq,
)
if not rows:
break
scanned += len(rows)
for event in reversed(rows):
if event.get("event_type") not in {"llm.ai.response", "ai_message"}:
continue
event_message_id = _event_message_id(event)
if event_message_id in message_ids and event_message_id not in found:
found[event_message_id] = event
seq_values = [row.get("seq") for row in rows if isinstance(row.get("seq"), int)]
if len(rows) < _MESSAGE_SCAN_BATCH or not seq_values:
break
before_seq = min(seq_values)
return found
async def _get_run_for_thread(
request: Request,
*,
thread_id: str,
run_id: str,
) -> dict[str, Any]:
run = await get_run_store(request).get(run_id)
if run is None or run.get("thread_id") != thread_id:
raise HTTPException(
status_code=404,
detail=f"Run {run_id} not found in thread {thread_id}",
)
return run
def _agent_id_from_records(
run: dict[str, Any],
thread: dict[str, Any] | None,
) -> str:
metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {}
candidates: list[Any] = [
run.get("assistant_id"),
metadata.get("agent_id"),
metadata.get("agent_name"),
]
if isinstance(thread, dict):
thread_metadata = thread.get("metadata")
if not isinstance(thread_metadata, dict):
thread_metadata = thread.get("metadata_json")
candidates.append(thread.get("assistant_id"))
if isinstance(thread_metadata, dict):
candidates.extend(
(
thread_metadata.get("agent_id"),
thread_metadata.get("agent_name"),
)
)
for candidate in candidates:
value = str(candidate or "").strip()
if value:
return value[:128]
return "lead_agent"
async def _resolve_agent_id(
request: Request,
*,
thread_id: str,
run: dict[str, Any],
user_id: str | None,
) -> str:
try:
thread = await get_thread_store(request).get(thread_id, user_id=user_id)
except Exception:
logger.debug("Unable to read thread metadata while resolving feedback agent", exc_info=True)
thread = None
return _agent_id_from_records(run, thread)
async def _resolve_message_context(
request: Request,
*,
thread_id: str,
message_id: str,
user_id: str | None,
) -> tuple[str, str]:
event = await _find_ai_message_event(
request,
thread_id=thread_id,
message_id=message_id,
)
run_id = str(event.get("run_id") or "").strip()
if not run_id:
raise HTTPException(status_code=409, detail="Assistant message has no run identity")
run = await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
agent_id = await _resolve_agent_id(
request,
thread_id=thread_id,
run=run,
user_id=user_id,
)
return run_id, agent_id
async def _write_feedback(
request: Request,
*,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
body: FeedbackWriteRequest,
user_id: str | None,
) -> dict[str, Any]:
return await get_feedback_repo(request).upsert(
run_id=run_id,
thread_id=thread_id,
message_id=message_id,
agent_id=agent_id,
rating=body.rating,
reasons=body.reason_values(),
comment=body.normalized_detail(),
user_id=user_id,
)
async def _upsert_run_scoped_feedback(
request: Request,
*,
thread_id: str,
run_id: str,
body: FeedbackWriteRequest,
) -> dict[str, Any]:
user_id = await get_current_user(request)
run = await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
agent_id = await _resolve_agent_id(
request,
thread_id=thread_id,
run=run,
user_id=user_id,
)
if body.message_id is not None:
resolved_run_id, resolved_agent_id = await _resolve_message_context(
request,
thread_id=thread_id,
message_id=body.message_id,
user_id=user_id,
)
if resolved_run_id != run_id:
raise HTTPException(status_code=409, detail="Message does not belong to this run")
agent_id = resolved_agent_id
return await _write_feedback(
request,
thread_id=thread_id,
run_id=run_id,
message_id=body.message_id,
agent_id=agent_id,
body=body,
user_id=user_id,
)
@router.post(
"/{thread_id}/feedback-contexts",
response_model=FeedbackContextsResponse,
)
@require_permission("threads", "read", owner_check=True, require_existing=True)
async def get_feedback_contexts(
thread_id: str,
body: FeedbackContextsRequest,
request: Request,
) -> dict[str, Any]:
"""Resolve visible assistant messages and their feedback in one request."""
user_id = await get_current_user(request)
events = await _find_ai_message_events(
request,
thread_id=thread_id,
message_ids=set(body.message_ids),
)
try:
thread = await get_thread_store(request).get(thread_id, user_id=user_id)
except Exception:
logger.debug("Unable to read thread metadata for feedback batch", exc_info=True)
thread = None
feedback_rows = await get_feedback_repo(request).list_by_message_ids(
thread_id=thread_id,
message_ids=body.message_ids,
user_id=user_id,
)
feedback_by_message = {
str(row["message_id"]): row
for row in feedback_rows
if row.get("message_id")
}
run_cache: dict[str, dict[str, Any] | None] = {}
contexts: list[dict[str, Any]] = []
unresolved: list[str] = []
run_store = get_run_store(request)
for message_id in body.message_ids:
event = events.get(message_id)
run_id = str((event or {}).get("run_id") or "").strip()
if not run_id:
unresolved.append(message_id)
continue
if run_id not in run_cache:
run_cache[run_id] = await run_store.get(run_id)
run = run_cache[run_id]
if run is None or run.get("thread_id") != thread_id:
unresolved.append(message_id)
continue
contexts.append(
{
"thread_id": thread_id,
"message_id": message_id,
"run_id": run_id,
"agent_id": _agent_id_from_records(run, thread),
"feedback": feedback_by_message.get(message_id),
}
)
return {
"contexts": contexts,
"unresolved_message_ids": unresolved,
}
@router.get(
"/{thread_id}/messages/{message_id}/feedback-context",
response_model=FeedbackContextResponse,
)
@require_permission("threads", "read", owner_check=True, require_existing=True)
async def get_message_feedback_context(
thread_id: str,
message_id: str,
request: Request,
) -> dict[str, Any]:
"""Resolve one assistant message to run/agent plus the caller's feedback."""
user_id = await get_current_user(request)
run_id, agent_id = await _resolve_message_context(
request,
thread_id=thread_id,
message_id=message_id,
user_id=user_id,
)
feedback = await get_feedback_repo(request).get_by_target(
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
user_id=user_id,
)
return {
"thread_id": thread_id,
"message_id": message_id,
"run_id": run_id,
"agent_id": agent_id,
"feedback": feedback,
}
@router.put(
"/{thread_id}/messages/{message_id}/feedback",
response_model=FeedbackResponse,
)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def upsert_message_feedback(
thread_id: str,
message_id: str,
body: FeedbackWriteRequest,
request: Request,
) -> dict[str, Any]:
"""Idempotently rate one exact assistant answer."""
if body.message_id is not None and body.message_id != message_id:
raise HTTPException(status_code=409, detail="Body message_id does not match URL")
user_id = await get_current_user(request)
run_id, agent_id = await _resolve_message_context(
request,
thread_id=thread_id,
message_id=message_id,
user_id=user_id,
)
return await _write_feedback(
request,
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
body=body,
user_id=user_id,
)
@router.delete("/{thread_id}/messages/{message_id}/feedback")
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_message_feedback(
thread_id: str,
message_id: str,
request: Request,
) -> dict[str, bool]:
user_id = await get_current_user(request)
run_id, agent_id = await _resolve_message_context(
request,
thread_id=thread_id,
message_id=message_id,
user_id=user_id,
)
deleted = await get_feedback_repo(request).delete_by_target(
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
user_id=user_id,
)
if not deleted:
raise HTTPException(status_code=404, detail="No feedback found for this message")
return {"success": True}
@router.put("/{thread_id}/runs/{run_id}/feedback", response_model=FeedbackResponse)
@@ -63,45 +547,92 @@ class FeedbackStatsResponse(BaseModel):
async def upsert_feedback(
thread_id: str,
run_id: str,
body: FeedbackUpsertRequest,
body: FeedbackWriteRequest,
request: Request,
) -> dict[str, Any]:
"""Create or update feedback for a run (idempotent)."""
if body.rating not in (1, -1):
raise HTTPException(status_code=400, detail="rating must be +1 or -1")
"""Compatibility run-scoped upsert; new clients should use message scope."""
user_id = await get_current_user(request)
run_store = get_run_store(request)
run = await run_store.get(run_id)
if run is None:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
if run.get("thread_id") != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found in thread {thread_id}")
feedback_repo = get_feedback_repo(request)
return await feedback_repo.upsert(
run_id=run_id,
return await _upsert_run_scoped_feedback(
request,
thread_id=thread_id,
rating=body.rating,
user_id=user_id,
comment=body.comment,
run_id=run_id,
body=body,
)
@router.post("/{thread_id}/runs/{run_id}/feedback", response_model=FeedbackResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def create_feedback(
thread_id: str,
run_id: str,
body: FeedbackWriteRequest,
request: Request,
) -> dict[str, Any]:
"""Backward-compatible idempotent alias of the run-scoped PUT."""
return await _upsert_run_scoped_feedback(
request,
thread_id=thread_id,
run_id=run_id,
body=body,
)
@router.get(
"/{thread_id}/runs/{run_id}/feedback",
response_model=list[FeedbackResponse],
)
@require_permission("threads", "read", owner_check=True)
async def list_feedback(
thread_id: str,
run_id: str,
request: Request,
message_id: str | None = Query(default=None, min_length=1, max_length=64),
) -> list[dict[str, Any]]:
"""List the caller's feedback for a run, optionally for one message."""
await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
user_id = await get_current_user(request)
return await get_feedback_repo(request).list_by_run(
thread_id,
run_id,
message_id=message_id,
user_id=user_id,
)
@router.get(
"/{thread_id}/runs/{run_id}/feedback/stats",
response_model=FeedbackStatsResponse,
)
@require_permission("threads", "read", owner_check=True)
async def feedback_stats(
thread_id: str,
run_id: str,
request: Request,
) -> dict[str, Any]:
"""Get positive/negative counts for one run."""
await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
return await get_feedback_repo(request).aggregate_by_run(thread_id, run_id)
@router.delete("/{thread_id}/runs/{run_id}/feedback")
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_run_feedback(
thread_id: str,
run_id: str,
request: Request,
message_id: str | None = Query(default=None, min_length=1, max_length=64),
) -> dict[str, bool]:
"""Delete the current user's feedback for a run."""
"""Delete caller feedback for one run or one message in that run."""
await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
user_id = await get_current_user(request)
feedback_repo = get_feedback_repo(request)
deleted = await feedback_repo.delete_by_run(
deleted = await get_feedback_repo(request).delete_by_run(
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
user_id=user_id,
)
if not deleted:
@@ -109,63 +640,6 @@ async def delete_run_feedback(
return {"success": True}
@router.post("/{thread_id}/runs/{run_id}/feedback", response_model=FeedbackResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def create_feedback(
thread_id: str,
run_id: str,
body: FeedbackCreateRequest,
request: Request,
) -> dict[str, Any]:
"""Submit feedback (thumbs-up/down) for a run."""
if body.rating not in (1, -1):
raise HTTPException(status_code=400, detail="rating must be +1 or -1")
user_id = await get_current_user(request)
# Validate run exists and belongs to thread
run_store = get_run_store(request)
run = await run_store.get(run_id)
if run is None:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found")
if run.get("thread_id") != thread_id:
raise HTTPException(status_code=404, detail=f"Run {run_id} not found in thread {thread_id}")
feedback_repo = get_feedback_repo(request)
return await feedback_repo.create(
run_id=run_id,
thread_id=thread_id,
rating=body.rating,
user_id=user_id,
message_id=body.message_id,
comment=body.comment,
)
@router.get("/{thread_id}/runs/{run_id}/feedback", response_model=list[FeedbackResponse])
@require_permission("threads", "read", owner_check=True)
async def list_feedback(
thread_id: str,
run_id: str,
request: Request,
) -> list[dict[str, Any]]:
"""List all feedback for a run."""
feedback_repo = get_feedback_repo(request)
return await feedback_repo.list_by_run(thread_id, run_id)
@router.get("/{thread_id}/runs/{run_id}/feedback/stats", response_model=FeedbackStatsResponse)
@require_permission("threads", "read", owner_check=True)
async def feedback_stats(
thread_id: str,
run_id: str,
request: Request,
) -> dict[str, Any]:
"""Get aggregated feedback stats (positive/negative counts) for a run."""
feedback_repo = get_feedback_repo(request)
return await feedback_repo.aggregate_by_run(thread_id, run_id)
@router.delete("/{thread_id}/runs/{run_id}/feedback/{feedback_id}")
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_feedback(
@@ -174,15 +648,34 @@ async def delete_feedback(
feedback_id: str,
request: Request,
) -> dict[str, bool]:
"""Delete a feedback record."""
feedback_repo = get_feedback_repo(request)
# Verify feedback belongs to the specified thread/run before deleting
existing = await feedback_repo.get(feedback_id)
if existing is None:
"""Delete one caller-owned feedback row after path-scope verification."""
user_id = await get_current_user(request)
existing = await get_feedback_repo(request).get(feedback_id, user_id=user_id)
if (
existing is None
or existing.get("thread_id") != thread_id
or existing.get("run_id") != run_id
):
raise HTTPException(status_code=404, detail=f"Feedback {feedback_id} not found")
if existing.get("thread_id") != thread_id or existing.get("run_id") != run_id:
raise HTTPException(status_code=404, detail=f"Feedback {feedback_id} not found in run {run_id}")
deleted = await feedback_repo.delete(feedback_id)
deleted = await get_feedback_repo(request).delete(feedback_id, user_id=user_id)
if not deleted:
raise HTTPException(status_code=404, detail=f"Feedback {feedback_id} not found")
return {"success": True}
@agent_score_router.get(
"/{agent_id}/feedback-score",
response_model=AgentFeedbackScoreResponse,
)
async def get_agent_feedback_score(
agent_id: str,
request: Request,
) -> dict[str, Any]:
"""Aggregate user-feedback answer quality for administrators only."""
await _require_admin(request)
normalized_agent_id = agent_id.strip()
if not normalized_agent_id or len(normalized_agent_id) > 128:
raise HTTPException(status_code=422, detail="Invalid agent_id")
return await get_feedback_repo(request).aggregate_by_agent(normalized_agent_id)