Files

682 lines
21 KiB
Python

"""Assistant-answer feedback and transparent user-feedback scoring APIs."""
from __future__ import annotations
import logging
from enum import StrEnum
from typing import Any, Literal
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field, model_validator
from app.gateway.authz import require_permission
from app.gateway.deps import (
get_current_user,
get_feedback_repo,
get_optional_user_from_request,
get_run_event_store,
get_run_store,
get_thread_store,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/threads", tags=["feedback"])
agent_score_router = APIRouter(prefix="/api/agents", tags=["feedback"])
_MESSAGE_SCAN_BATCH = 200
_MESSAGE_SCAN_MAX = 10_000
async def _require_admin(request: Request) -> None:
"""Apply the project's admin role rule, preserving auth-disabled mode."""
user = await get_optional_user_from_request(request)
if user is None:
return
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="Agent feedback score is admin-only")
class FeedbackReason(StrEnum):
INCORRECT_OR_INCOMPLETE = "incorrect_or_incomplete"
UNMET_EXPECTATIONS = "unmet_expectations"
SLOW_OR_PROBLEMATIC = "slow_or_problematic"
STYLE_OR_TONE = "style_or_tone"
SAFETY_OR_LEGAL = "safety_or_legal"
OTHER = "other"
class FeedbackWriteRequest(BaseModel):
rating: Literal[-1, 1] = Field(
...,
description="+1 (excellent answer) or -1 (poor answer)",
)
reasons: list[FeedbackReason] = Field(default_factory=list, max_length=6)
detail: str | None = Field(default=None, max_length=2000)
# Kept for existing API clients. New clients should use ``detail``.
comment: str | None = Field(
default=None,
max_length=2000,
description="Deprecated compatibility alias for detail",
)
# Used only by the legacy run-scoped routes. Message-scoped routes take
# this value from the URL and reject a forged run/message association.
message_id: str | None = Field(default=None, min_length=1, max_length=64)
@model_validator(mode="after")
def validate_feedback_shape(self) -> FeedbackWriteRequest:
self.reasons = list(dict.fromkeys(self.reasons))
if self.rating == 1 and self.reasons:
raise ValueError("positive feedback must not include problem reasons")
if self.rating == -1 and not self.reasons:
raise ValueError("negative feedback requires at least one problem reason")
if self.detail is not None and self.comment is not None and self.detail != self.comment:
raise ValueError("detail and deprecated comment must match when both are provided")
return self
def normalized_detail(self) -> str | None:
value = self.detail if self.detail is not None else self.comment
if value is None:
return None
stripped = value.strip()
return stripped or None
def reason_values(self) -> list[str]:
return [reason.value for reason in self.reasons]
# Compatibility names used by OpenAPI clients generated from the prior API.
FeedbackCreateRequest = FeedbackWriteRequest
FeedbackUpsertRequest = FeedbackWriteRequest
class FeedbackResponse(BaseModel):
feedback_id: str
run_id: str
thread_id: str
user_id: str | None = None
message_id: str | None = None
agent_id: str
rating: Literal[-1, 1]
reasons: list[FeedbackReason] = Field(default_factory=list)
detail: str | None = None
comment: str | None = None
created_at: str = ""
updated_at: str = ""
class FeedbackStatsResponse(BaseModel):
run_id: str
total: int = 0
positive: int = 0
negative: int = 0
class FeedbackContextResponse(BaseModel):
thread_id: str
message_id: str
run_id: str
agent_id: str
feedback: FeedbackResponse | None = None
class FeedbackContextsRequest(BaseModel):
message_ids: list[str] = Field(..., min_length=1, max_length=200)
@model_validator(mode="after")
def normalize_message_ids(self) -> FeedbackContextsRequest:
normalized: list[str] = []
for item in self.message_ids:
value = str(item).strip()
if not value or len(value) > 64:
raise ValueError("message_ids must contain non-empty ids up to 64 characters")
if value not in normalized:
normalized.append(value)
self.message_ids = normalized
return self
class FeedbackContextsResponse(BaseModel):
contexts: list[FeedbackContextResponse]
unresolved_message_ids: list[str]
class ScoreBandResponse(BaseModel):
key: str
label: str
min: float
max: float
class DimensionScoreResponse(BaseModel):
weight: float
score: float | None
evidence_count: int
positive_count: int
negative_count: int
mapped_reasons: list[FeedbackReason]
class ReasonBreakdownResponse(BaseModel):
count: int
rate: float
class AgentFeedbackScoreResponse(BaseModel):
agent_id: str
score: float | None
score_band: ScoreBandResponse
confidence: float = Field(..., ge=0, le=1)
confidence_level: Literal["none", "low", "medium", "high"]
sample_count: int
positive_count: int
negative_count: int
task_success_rate: float | None
task_score: float | None
dimensions: dict[str, DimensionScoreResponse]
reason_breakdown: dict[str, ReasonBreakdownResponse]
prior: dict[str, int | float]
formula: dict[str, str]
def _event_message_id(event: dict[str, Any]) -> str | None:
content = event.get("content")
if not isinstance(content, dict):
return None
value = content.get("id")
return str(value) if value else None
async def _find_ai_message_event(
request: Request,
*,
thread_id: str,
message_id: str,
) -> dict[str, Any]:
"""Resolve a checkpoint message id to its exact durable run event."""
found = await _find_ai_message_events(
request,
thread_id=thread_id,
message_ids={message_id},
)
if message_id in found:
return found[message_id]
raise HTTPException(
status_code=404,
detail=f"Assistant message {message_id} was not found in thread {thread_id}",
)
async def _find_ai_message_events(
request: Request,
*,
thread_id: str,
message_ids: set[str],
) -> dict[str, dict[str, Any]]:
"""Resolve up to 200 message ids in one bounded reverse event scan."""
event_store = get_run_event_store(request)
before_seq: int | None = None
scanned = 0
found: dict[str, dict[str, Any]] = {}
while scanned < _MESSAGE_SCAN_MAX and len(found) < len(message_ids):
rows = await event_store.list_messages(
thread_id,
limit=_MESSAGE_SCAN_BATCH,
before_seq=before_seq,
)
if not rows:
break
scanned += len(rows)
for event in reversed(rows):
if event.get("event_type") not in {"llm.ai.response", "ai_message"}:
continue
event_message_id = _event_message_id(event)
if event_message_id in message_ids and event_message_id not in found:
found[event_message_id] = event
seq_values = [row.get("seq") for row in rows if isinstance(row.get("seq"), int)]
if len(rows) < _MESSAGE_SCAN_BATCH or not seq_values:
break
before_seq = min(seq_values)
return found
async def _get_run_for_thread(
request: Request,
*,
thread_id: str,
run_id: str,
) -> dict[str, Any]:
run = await get_run_store(request).get(run_id)
if run is None or run.get("thread_id") != thread_id:
raise HTTPException(
status_code=404,
detail=f"Run {run_id} not found in thread {thread_id}",
)
return run
def _agent_id_from_records(
run: dict[str, Any],
thread: dict[str, Any] | None,
) -> str:
metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {}
candidates: list[Any] = [
run.get("assistant_id"),
metadata.get("agent_id"),
metadata.get("agent_name"),
]
if isinstance(thread, dict):
thread_metadata = thread.get("metadata")
if not isinstance(thread_metadata, dict):
thread_metadata = thread.get("metadata_json")
candidates.append(thread.get("assistant_id"))
if isinstance(thread_metadata, dict):
candidates.extend(
(
thread_metadata.get("agent_id"),
thread_metadata.get("agent_name"),
)
)
for candidate in candidates:
value = str(candidate or "").strip()
if value:
return value[:128]
return "lead_agent"
async def _resolve_agent_id(
request: Request,
*,
thread_id: str,
run: dict[str, Any],
user_id: str | None,
) -> str:
try:
thread = await get_thread_store(request).get(thread_id, user_id=user_id)
except Exception:
logger.debug("Unable to read thread metadata while resolving feedback agent", exc_info=True)
thread = None
return _agent_id_from_records(run, thread)
async def _resolve_message_context(
request: Request,
*,
thread_id: str,
message_id: str,
user_id: str | None,
) -> tuple[str, str]:
event = await _find_ai_message_event(
request,
thread_id=thread_id,
message_id=message_id,
)
run_id = str(event.get("run_id") or "").strip()
if not run_id:
raise HTTPException(status_code=409, detail="Assistant message has no run identity")
run = await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
agent_id = await _resolve_agent_id(
request,
thread_id=thread_id,
run=run,
user_id=user_id,
)
return run_id, agent_id
async def _write_feedback(
request: Request,
*,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
body: FeedbackWriteRequest,
user_id: str | None,
) -> dict[str, Any]:
return await get_feedback_repo(request).upsert(
run_id=run_id,
thread_id=thread_id,
message_id=message_id,
agent_id=agent_id,
rating=body.rating,
reasons=body.reason_values(),
comment=body.normalized_detail(),
user_id=user_id,
)
async def _upsert_run_scoped_feedback(
request: Request,
*,
thread_id: str,
run_id: str,
body: FeedbackWriteRequest,
) -> dict[str, Any]:
user_id = await get_current_user(request)
run = await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
agent_id = await _resolve_agent_id(
request,
thread_id=thread_id,
run=run,
user_id=user_id,
)
if body.message_id is not None:
resolved_run_id, resolved_agent_id = await _resolve_message_context(
request,
thread_id=thread_id,
message_id=body.message_id,
user_id=user_id,
)
if resolved_run_id != run_id:
raise HTTPException(status_code=409, detail="Message does not belong to this run")
agent_id = resolved_agent_id
return await _write_feedback(
request,
thread_id=thread_id,
run_id=run_id,
message_id=body.message_id,
agent_id=agent_id,
body=body,
user_id=user_id,
)
@router.post(
"/{thread_id}/feedback-contexts",
response_model=FeedbackContextsResponse,
)
@require_permission("threads", "read", owner_check=True, require_existing=True)
async def get_feedback_contexts(
thread_id: str,
body: FeedbackContextsRequest,
request: Request,
) -> dict[str, Any]:
"""Resolve visible assistant messages and their feedback in one request."""
user_id = await get_current_user(request)
events = await _find_ai_message_events(
request,
thread_id=thread_id,
message_ids=set(body.message_ids),
)
try:
thread = await get_thread_store(request).get(thread_id, user_id=user_id)
except Exception:
logger.debug("Unable to read thread metadata for feedback batch", exc_info=True)
thread = None
feedback_rows = await get_feedback_repo(request).list_by_message_ids(
thread_id=thread_id,
message_ids=body.message_ids,
user_id=user_id,
)
feedback_by_message = {
str(row["message_id"]): row
for row in feedback_rows
if row.get("message_id")
}
run_cache: dict[str, dict[str, Any] | None] = {}
contexts: list[dict[str, Any]] = []
unresolved: list[str] = []
run_store = get_run_store(request)
for message_id in body.message_ids:
event = events.get(message_id)
run_id = str((event or {}).get("run_id") or "").strip()
if not run_id:
unresolved.append(message_id)
continue
if run_id not in run_cache:
run_cache[run_id] = await run_store.get(run_id)
run = run_cache[run_id]
if run is None or run.get("thread_id") != thread_id:
unresolved.append(message_id)
continue
contexts.append(
{
"thread_id": thread_id,
"message_id": message_id,
"run_id": run_id,
"agent_id": _agent_id_from_records(run, thread),
"feedback": feedback_by_message.get(message_id),
}
)
return {
"contexts": contexts,
"unresolved_message_ids": unresolved,
}
@router.get(
"/{thread_id}/messages/{message_id}/feedback-context",
response_model=FeedbackContextResponse,
)
@require_permission("threads", "read", owner_check=True, require_existing=True)
async def get_message_feedback_context(
thread_id: str,
message_id: str,
request: Request,
) -> dict[str, Any]:
"""Resolve one assistant message to run/agent plus the caller's feedback."""
user_id = await get_current_user(request)
run_id, agent_id = await _resolve_message_context(
request,
thread_id=thread_id,
message_id=message_id,
user_id=user_id,
)
feedback = await get_feedback_repo(request).get_by_target(
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
user_id=user_id,
)
return {
"thread_id": thread_id,
"message_id": message_id,
"run_id": run_id,
"agent_id": agent_id,
"feedback": feedback,
}
@router.put(
"/{thread_id}/messages/{message_id}/feedback",
response_model=FeedbackResponse,
)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def upsert_message_feedback(
thread_id: str,
message_id: str,
body: FeedbackWriteRequest,
request: Request,
) -> dict[str, Any]:
"""Idempotently rate one exact assistant answer."""
if body.message_id is not None and body.message_id != message_id:
raise HTTPException(status_code=409, detail="Body message_id does not match URL")
user_id = await get_current_user(request)
run_id, agent_id = await _resolve_message_context(
request,
thread_id=thread_id,
message_id=message_id,
user_id=user_id,
)
return await _write_feedback(
request,
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
body=body,
user_id=user_id,
)
@router.delete("/{thread_id}/messages/{message_id}/feedback")
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_message_feedback(
thread_id: str,
message_id: str,
request: Request,
) -> dict[str, bool]:
user_id = await get_current_user(request)
run_id, agent_id = await _resolve_message_context(
request,
thread_id=thread_id,
message_id=message_id,
user_id=user_id,
)
deleted = await get_feedback_repo(request).delete_by_target(
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
user_id=user_id,
)
if not deleted:
raise HTTPException(status_code=404, detail="No feedback found for this message")
return {"success": True}
@router.put("/{thread_id}/runs/{run_id}/feedback", response_model=FeedbackResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def upsert_feedback(
thread_id: str,
run_id: str,
body: FeedbackWriteRequest,
request: Request,
) -> dict[str, Any]:
"""Compatibility run-scoped upsert; new clients should use message scope."""
return await _upsert_run_scoped_feedback(
request,
thread_id=thread_id,
run_id=run_id,
body=body,
)
@router.post("/{thread_id}/runs/{run_id}/feedback", response_model=FeedbackResponse)
@require_permission("threads", "write", owner_check=True, require_existing=True)
async def create_feedback(
thread_id: str,
run_id: str,
body: FeedbackWriteRequest,
request: Request,
) -> dict[str, Any]:
"""Backward-compatible idempotent alias of the run-scoped PUT."""
return await _upsert_run_scoped_feedback(
request,
thread_id=thread_id,
run_id=run_id,
body=body,
)
@router.get(
"/{thread_id}/runs/{run_id}/feedback",
response_model=list[FeedbackResponse],
)
@require_permission("threads", "read", owner_check=True)
async def list_feedback(
thread_id: str,
run_id: str,
request: Request,
message_id: str | None = Query(default=None, min_length=1, max_length=64),
) -> list[dict[str, Any]]:
"""List the caller's feedback for a run, optionally for one message."""
await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
user_id = await get_current_user(request)
return await get_feedback_repo(request).list_by_run(
thread_id,
run_id,
message_id=message_id,
user_id=user_id,
)
@router.get(
"/{thread_id}/runs/{run_id}/feedback/stats",
response_model=FeedbackStatsResponse,
)
@require_permission("threads", "read", owner_check=True)
async def feedback_stats(
thread_id: str,
run_id: str,
request: Request,
) -> dict[str, Any]:
"""Get positive/negative counts for one run."""
await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
return await get_feedback_repo(request).aggregate_by_run(thread_id, run_id)
@router.delete("/{thread_id}/runs/{run_id}/feedback")
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_run_feedback(
thread_id: str,
run_id: str,
request: Request,
message_id: str | None = Query(default=None, min_length=1, max_length=64),
) -> dict[str, bool]:
"""Delete caller feedback for one run or one message in that run."""
await _get_run_for_thread(request, thread_id=thread_id, run_id=run_id)
user_id = await get_current_user(request)
deleted = await get_feedback_repo(request).delete_by_run(
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
user_id=user_id,
)
if not deleted:
raise HTTPException(status_code=404, detail="No feedback found for this run")
return {"success": True}
@router.delete("/{thread_id}/runs/{run_id}/feedback/{feedback_id}")
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_feedback(
thread_id: str,
run_id: str,
feedback_id: str,
request: Request,
) -> dict[str, bool]:
"""Delete one caller-owned feedback row after path-scope verification."""
user_id = await get_current_user(request)
existing = await get_feedback_repo(request).get(feedback_id, user_id=user_id)
if (
existing is None
or existing.get("thread_id") != thread_id
or existing.get("run_id") != run_id
):
raise HTTPException(status_code=404, detail=f"Feedback {feedback_id} not found")
deleted = await get_feedback_repo(request).delete(feedback_id, user_id=user_id)
if not deleted:
raise HTTPException(status_code=404, detail=f"Feedback {feedback_id} not found")
return {"success": True}
@agent_score_router.get(
"/{agent_id}/feedback-score",
response_model=AgentFeedbackScoreResponse,
)
async def get_agent_feedback_score(
agent_id: str,
request: Request,
) -> dict[str, Any]:
"""Aggregate user-feedback answer quality for administrators only."""
await _require_admin(request)
normalized_agent_id = agent_id.strip()
if not normalized_agent_id or len(normalized_agent_id) > 128:
raise HTTPException(status_code=422, detail="Invalid agent_id")
return await get_feedback_repo(request).aggregate_by_agent(normalized_agent_id)