"""Assistant-answer feedback and transparent user-feedback scoring APIs.""" from __future__ import annotations import logging from enum import StrEnum from typing import Any, Literal from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel, Field, model_validator from app.gateway.authz import require_permission from app.gateway.deps import ( get_current_user, get_feedback_repo, get_optional_user_from_request, get_run_event_store, get_run_store, get_thread_store, ) logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/threads", tags=["feedback"]) agent_score_router = APIRouter(prefix="/api/agents", tags=["feedback"]) _MESSAGE_SCAN_BATCH = 200 _MESSAGE_SCAN_MAX = 10_000 async def _require_admin(request: Request) -> None: """Apply the project's admin role rule, preserving auth-disabled mode.""" user = await get_optional_user_from_request(request) if user is None: return if getattr(user, "system_role", None) != "admin": raise HTTPException(status_code=403, detail="Agent feedback score is admin-only") class FeedbackReason(StrEnum): INCORRECT_OR_INCOMPLETE = "incorrect_or_incomplete" UNMET_EXPECTATIONS = "unmet_expectations" SLOW_OR_PROBLEMATIC = "slow_or_problematic" STYLE_OR_TONE = "style_or_tone" SAFETY_OR_LEGAL = "safety_or_legal" OTHER = "other" class FeedbackWriteRequest(BaseModel): rating: Literal[-1, 1] = Field( ..., description="+1 (excellent answer) or -1 (poor answer)", ) reasons: list[FeedbackReason] = Field(default_factory=list, max_length=6) detail: str | None = Field(default=None, max_length=2000) # Kept for existing API clients. New clients should use ``detail``. comment: str | None = Field( default=None, max_length=2000, description="Deprecated compatibility alias for detail", ) # Used only by the legacy run-scoped routes. Message-scoped routes take # this value from the URL and reject a forged run/message association. message_id: str | None = Field(default=None, min_length=1, max_length=64) @model_validator(mode="after") def validate_feedback_shape(self) -> FeedbackWriteRequest: self.reasons = list(dict.fromkeys(self.reasons)) if self.rating == 1 and self.reasons: raise ValueError("positive feedback must not include problem reasons") if self.rating == -1 and not self.reasons: raise ValueError("negative feedback requires at least one problem reason") if self.detail is not None and self.comment is not None and self.detail != self.comment: raise ValueError("detail and deprecated comment must match when both are provided") return self def normalized_detail(self) -> str | None: value = self.detail if self.detail is not None else self.comment if value is None: return None stripped = value.strip() return stripped or None def reason_values(self) -> list[str]: return [reason.value for reason in self.reasons] # Compatibility names used by OpenAPI clients generated from the prior API. FeedbackCreateRequest = FeedbackWriteRequest FeedbackUpsertRequest = FeedbackWriteRequest class FeedbackResponse(BaseModel): feedback_id: str run_id: str thread_id: str user_id: str | None = None message_id: str | None = None agent_id: str rating: Literal[-1, 1] reasons: list[FeedbackReason] = Field(default_factory=list) detail: str | None = None comment: str | None = None created_at: str = "" updated_at: str = "" class FeedbackStatsResponse(BaseModel): run_id: str total: int = 0 positive: int = 0 negative: int = 0 class FeedbackContextResponse(BaseModel): thread_id: str message_id: str run_id: str agent_id: str feedback: FeedbackResponse | None = None class FeedbackContextsRequest(BaseModel): message_ids: list[str] = Field(..., min_length=1, max_length=200) @model_validator(mode="after") def normalize_message_ids(self) -> FeedbackContextsRequest: normalized: list[str] = [] for item in self.message_ids: value = str(item).strip() if not value or len(value) > 64: raise ValueError("message_ids must contain non-empty ids up to 64 characters") if value not in normalized: normalized.append(value) self.message_ids = normalized return self class FeedbackContextsResponse(BaseModel): contexts: list[FeedbackContextResponse] unresolved_message_ids: list[str] class ScoreBandResponse(BaseModel): key: str label: str min: float max: float class DimensionScoreResponse(BaseModel): weight: float score: float | None evidence_count: int positive_count: int negative_count: int mapped_reasons: list[FeedbackReason] class ReasonBreakdownResponse(BaseModel): count: int rate: float class AgentFeedbackScoreResponse(BaseModel): agent_id: str score: float | None score_band: ScoreBandResponse confidence: float = Field(..., ge=0, le=1) confidence_level: Literal["none", "low", "medium", "high"] sample_count: int positive_count: int negative_count: int task_success_rate: float | None task_score: float | None dimensions: dict[str, DimensionScoreResponse] reason_breakdown: dict[str, ReasonBreakdownResponse] prior: dict[str, int | float] formula: dict[str, str] def _event_message_id(event: dict[str, Any]) -> str | None: content = event.get("content") if not isinstance(content, dict): return None value = content.get("id") return str(value) if value else None async def _find_ai_message_event( request: Request, *, thread_id: str, message_id: str, ) -> dict[str, Any]: """Resolve a checkpoint message id to its exact durable run event.""" found = await _find_ai_message_events( request, thread_id=thread_id, message_ids={message_id}, ) if message_id in found: return found[message_id] raise HTTPException( status_code=404, detail=f"Assistant message {message_id} was not found in thread {thread_id}", ) async def _find_ai_message_events( request: Request, *, thread_id: str, message_ids: set[str], ) -> dict[str, dict[str, Any]]: """Resolve up to 200 message ids in one bounded reverse event scan.""" event_store = get_run_event_store(request) before_seq: int | None = None scanned = 0 found: dict[str, dict[str, Any]] = {} while scanned < _MESSAGE_SCAN_MAX and len(found) < len(message_ids): rows = await event_store.list_messages( thread_id, limit=_MESSAGE_SCAN_BATCH, before_seq=before_seq, ) if not rows: break scanned += len(rows) for event in reversed(rows): if event.get("event_type") not in {"llm.ai.response", "ai_message"}: continue event_message_id = _event_message_id(event) if event_message_id in message_ids and event_message_id not in found: found[event_message_id] = event seq_values = [row.get("seq") for row in rows if isinstance(row.get("seq"), int)] if len(rows) < _MESSAGE_SCAN_BATCH or not seq_values: break before_seq = min(seq_values) return found async def _get_run_for_thread( request: Request, *, thread_id: str, run_id: str, ) -> dict[str, Any]: run = await get_run_store(request).get(run_id) if run is None or run.get("thread_id") != thread_id: raise HTTPException( status_code=404, detail=f"Run {run_id} not found in thread {thread_id}", ) return run def _agent_id_from_records( run: dict[str, Any], thread: dict[str, Any] | None, ) -> str: metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {} candidates: list[Any] = [ run.get("assistant_id"), metadata.get("agent_id"), metadata.get("agent_name"), ] if isinstance(thread, dict): thread_metadata = thread.get("metadata") if not isinstance(thread_metadata, dict): thread_metadata = thread.get("metadata_json") candidates.append(thread.get("assistant_id")) if isinstance(thread_metadata, dict): candidates.extend( ( thread_metadata.get("agent_id"), thread_metadata.get("agent_name"), ) ) for candidate in candidates: value = str(candidate or "").strip() if value: return value[:128] return "lead_agent" async def _resolve_agent_id( request: Request, *, thread_id: str, run: dict[str, Any], user_id: str | None, ) -> str: try: thread = await get_thread_store(request).get(thread_id, user_id=user_id) except Exception: logger.debug("Unable to read thread metadata while resolving feedback agent", exc_info=True) thread = None return _agent_id_from_records(run, thread) async def _resolve_message_context( request: Request, *, thread_id: str, message_id: str, user_id: str | None, ) -> tuple[str, str]: event = await _find_ai_message_event( request, thread_id=thread_id, message_id=message_id, ) run_id = str(event.get("run_id") or "").strip() if not run_id: raise HTTPException(status_code=409, detail="Assistant message has no run identity") run = await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id) agent_id = await _resolve_agent_id( request, thread_id=thread_id, run=run, user_id=user_id, ) return run_id, agent_id async def _write_feedback( request: Request, *, thread_id: str, run_id: str, message_id: str | None, agent_id: str, body: FeedbackWriteRequest, user_id: str | None, ) -> dict[str, Any]: return await get_feedback_repo(request).upsert( run_id=run_id, thread_id=thread_id, message_id=message_id, agent_id=agent_id, rating=body.rating, reasons=body.reason_values(), comment=body.normalized_detail(), user_id=user_id, ) async def _upsert_run_scoped_feedback( request: Request, *, thread_id: str, run_id: str, body: FeedbackWriteRequest, ) -> dict[str, Any]: user_id = await get_current_user(request) run = await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id) agent_id = await _resolve_agent_id( request, thread_id=thread_id, run=run, user_id=user_id, ) if body.message_id is not None: resolved_run_id, resolved_agent_id = await _resolve_message_context( request, thread_id=thread_id, message_id=body.message_id, user_id=user_id, ) if resolved_run_id != run_id: raise HTTPException(status_code=409, detail="Message does not belong to this run") agent_id = resolved_agent_id return await _write_feedback( request, thread_id=thread_id, run_id=run_id, message_id=body.message_id, agent_id=agent_id, body=body, user_id=user_id, ) @router.post( "/{thread_id}/feedback-contexts", response_model=FeedbackContextsResponse, ) @require_permission("threads", "read", owner_check=True, require_existing=True) async def get_feedback_contexts( thread_id: str, body: FeedbackContextsRequest, request: Request, ) -> dict[str, Any]: """Resolve visible assistant messages and their feedback in one request.""" user_id = await get_current_user(request) events = await _find_ai_message_events( request, thread_id=thread_id, message_ids=set(body.message_ids), ) try: thread = await get_thread_store(request).get(thread_id, user_id=user_id) except Exception: logger.debug("Unable to read thread metadata for feedback batch", exc_info=True) thread = None feedback_rows = await get_feedback_repo(request).list_by_message_ids( thread_id=thread_id, message_ids=body.message_ids, user_id=user_id, ) feedback_by_message = { str(row["message_id"]): row for row in feedback_rows if row.get("message_id") } run_cache: dict[str, dict[str, Any] | None] = {} contexts: list[dict[str, Any]] = [] unresolved: list[str] = [] run_store = get_run_store(request) for message_id in body.message_ids: event = events.get(message_id) run_id = str((event or {}).get("run_id") or "").strip() if not run_id: unresolved.append(message_id) continue if run_id not in run_cache: run_cache[run_id] = await run_store.get(run_id) run = run_cache[run_id] if run is None or run.get("thread_id") != thread_id: unresolved.append(message_id) continue contexts.append( { "thread_id": thread_id, "message_id": message_id, "run_id": run_id, "agent_id": _agent_id_from_records(run, thread), "feedback": feedback_by_message.get(message_id), } ) return { "contexts": contexts, "unresolved_message_ids": unresolved, } @router.get( "/{thread_id}/messages/{message_id}/feedback-context", response_model=FeedbackContextResponse, ) @require_permission("threads", "read", owner_check=True, require_existing=True) async def get_message_feedback_context( thread_id: str, message_id: str, request: Request, ) -> dict[str, Any]: """Resolve one assistant message to run/agent plus the caller's feedback.""" user_id = await get_current_user(request) run_id, agent_id = await _resolve_message_context( request, thread_id=thread_id, message_id=message_id, user_id=user_id, ) feedback = await get_feedback_repo(request).get_by_target( thread_id=thread_id, run_id=run_id, message_id=message_id, agent_id=agent_id, user_id=user_id, ) return { "thread_id": thread_id, "message_id": message_id, "run_id": run_id, "agent_id": agent_id, "feedback": feedback, } @router.put( "/{thread_id}/messages/{message_id}/feedback", response_model=FeedbackResponse, ) @require_permission("threads", "write", owner_check=True, require_existing=True) async def upsert_message_feedback( thread_id: str, message_id: str, body: FeedbackWriteRequest, request: Request, ) -> dict[str, Any]: """Idempotently rate one exact assistant answer.""" if body.message_id is not None and body.message_id != message_id: raise HTTPException(status_code=409, detail="Body message_id does not match URL") user_id = await get_current_user(request) run_id, agent_id = await _resolve_message_context( request, thread_id=thread_id, message_id=message_id, user_id=user_id, ) return await _write_feedback( request, thread_id=thread_id, run_id=run_id, message_id=message_id, agent_id=agent_id, body=body, user_id=user_id, ) @router.delete("/{thread_id}/messages/{message_id}/feedback") @require_permission("threads", "delete", owner_check=True, require_existing=True) async def delete_message_feedback( thread_id: str, message_id: str, request: Request, ) -> dict[str, bool]: user_id = await get_current_user(request) run_id, agent_id = await _resolve_message_context( request, thread_id=thread_id, message_id=message_id, user_id=user_id, ) deleted = await get_feedback_repo(request).delete_by_target( thread_id=thread_id, run_id=run_id, message_id=message_id, agent_id=agent_id, user_id=user_id, ) if not deleted: raise HTTPException(status_code=404, detail="No feedback found for this message") return {"success": True} @router.put("/{thread_id}/runs/{run_id}/feedback", response_model=FeedbackResponse) @require_permission("threads", "write", owner_check=True, require_existing=True) async def upsert_feedback( thread_id: str, run_id: str, body: FeedbackWriteRequest, request: Request, ) -> dict[str, Any]: """Compatibility run-scoped upsert; new clients should use message scope.""" return await _upsert_run_scoped_feedback( request, thread_id=thread_id, run_id=run_id, body=body, ) @router.post("/{thread_id}/runs/{run_id}/feedback", response_model=FeedbackResponse) @require_permission("threads", "write", owner_check=True, require_existing=True) async def create_feedback( thread_id: str, run_id: str, body: FeedbackWriteRequest, request: Request, ) -> dict[str, Any]: """Backward-compatible idempotent alias of the run-scoped PUT.""" return await _upsert_run_scoped_feedback( request, thread_id=thread_id, run_id=run_id, body=body, ) @router.get( "/{thread_id}/runs/{run_id}/feedback", response_model=list[FeedbackResponse], ) @require_permission("threads", "read", owner_check=True) async def list_feedback( thread_id: str, run_id: str, request: Request, message_id: str | None = Query(default=None, min_length=1, max_length=64), ) -> list[dict[str, Any]]: """List the caller's feedback for a run, optionally for one message.""" await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id) user_id = await get_current_user(request) return await get_feedback_repo(request).list_by_run( thread_id, run_id, message_id=message_id, user_id=user_id, ) @router.get( "/{thread_id}/runs/{run_id}/feedback/stats", response_model=FeedbackStatsResponse, ) @require_permission("threads", "read", owner_check=True) async def feedback_stats( thread_id: str, run_id: str, request: Request, ) -> dict[str, Any]: """Get positive/negative counts for one run.""" await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id) return await get_feedback_repo(request).aggregate_by_run(thread_id, run_id) @router.delete("/{thread_id}/runs/{run_id}/feedback") @require_permission("threads", "delete", owner_check=True, require_existing=True) async def delete_run_feedback( thread_id: str, run_id: str, request: Request, message_id: str | None = Query(default=None, min_length=1, max_length=64), ) -> dict[str, bool]: """Delete caller feedback for one run or one message in that run.""" await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id) user_id = await get_current_user(request) deleted = await get_feedback_repo(request).delete_by_run( thread_id=thread_id, run_id=run_id, message_id=message_id, user_id=user_id, ) if not deleted: raise HTTPException(status_code=404, detail="No feedback found for this run") return {"success": True} @router.delete("/{thread_id}/runs/{run_id}/feedback/{feedback_id}") @require_permission("threads", "delete", owner_check=True, require_existing=True) async def delete_feedback( thread_id: str, run_id: str, feedback_id: str, request: Request, ) -> dict[str, bool]: """Delete one caller-owned feedback row after path-scope verification.""" user_id = await get_current_user(request) existing = await get_feedback_repo(request).get(feedback_id, user_id=user_id) if ( existing is None or existing.get("thread_id") != thread_id or existing.get("run_id") != run_id ): raise HTTPException(status_code=404, detail=f"Feedback {feedback_id} not found") deleted = await get_feedback_repo(request).delete(feedback_id, user_id=user_id) if not deleted: raise HTTPException(status_code=404, detail=f"Feedback {feedback_id} not found") return {"success": True} @agent_score_router.get( "/{agent_id}/feedback-score", response_model=AgentFeedbackScoreResponse, ) async def get_agent_feedback_score( agent_id: str, request: Request, ) -> dict[str, Any]: """Aggregate user-feedback answer quality for administrators only.""" await _require_admin(request) normalized_agent_id = agent_id.strip() if not normalized_agent_id or len(normalized_agent_id) > 128: raise HTTPException(status_code=422, detail="Invalid agent_id") return await get_feedback_repo(request).aggregate_by_agent(normalized_agent_id)