sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+38
-24
@@ -1,38 +1,52 @@
|
||||
"""ORM model for user feedback on runs."""
|
||||
"""ORM model for user feedback on assistant answers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
# Prefix-unique instead of UniqueConstraint: a plain unique over three
|
||||
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
|
||||
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
|
||||
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
|
||||
# mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
# ``target_key`` is a SHA-256 digest of user/thread/message/agent. A
|
||||
# digest keeps the idempotency index well below MySQL's legacy utf8mb4
|
||||
# key-size limit; indexing all four source strings would exceed it.
|
||||
__table_args__ = (
|
||||
Index("uq_feedback_target_key", "target_key", unique=True),
|
||||
Index("ix_feedback_agent_created", "agent_id", "created_at"),
|
||||
)
|
||||
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# message_id is an optional RunEventStore event identifier —
|
||||
# allows feedback to target a specific message or the entire run
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# ``message_id`` is the LangChain/checkpoint message id serialized inside
|
||||
# a RunEventStore ``llm.ai.response`` event. It stays nullable solely for
|
||||
# legacy run-level feedback rows created before answer-level feedback.
|
||||
|
||||
agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="lead_agent")
|
||||
target_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
|
||||
rating: Mapped[int] = mapped_column(nullable=False)
|
||||
# +1 (thumbs-up) or -1 (thumbs-down)
|
||||
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional text feedback from the user
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional user detail. The API exposes this as ``detail`` and retains
|
||||
# ``comment`` as a compatibility alias for older clients.
|
||||
|
||||
reasons: Mapped[list[str]] = mapped_column(PortableJSON(), default=list)
|
||||
# Stable machine-readable reason codes for negative feedback.
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"""Transparent user-feedback scoring for assistant answers.
|
||||
|
||||
This module deliberately uses no model or heuristic content grading. It
|
||||
summarizes explicit thumbs feedback and selected problem reasons only. The
|
||||
result is an answer-quality signal, not a complete evaluation of an agent's
|
||||
planning, reasoning trajectory, tool use, latency, reliability, or safety.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
FEEDBACK_REASONS = (
|
||||
"incorrect_or_incomplete",
|
||||
"unmet_expectations",
|
||||
"slow_or_problematic",
|
||||
"style_or_tone",
|
||||
"safety_or_legal",
|
||||
"other",
|
||||
)
|
||||
|
||||
# The five response-quality dimensions and weights are fixed product policy.
|
||||
# A negative reason is evidence only for the explicitly mapped dimensions;
|
||||
# operational, communication, safety, and uncategorized complaints remain in
|
||||
# the problem breakdown instead of being mislabeled as task-quality evidence.
|
||||
DIMENSION_CONFIG: dict[str, dict[str, Any]] = {
|
||||
"correctness": {
|
||||
"weight": 0.30,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"completeness": {
|
||||
"weight": 0.20,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"evidence_grounding": {
|
||||
"weight": 0.20,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"relevance": {
|
||||
"weight": 0.15,
|
||||
"mapped_reasons": ["unmet_expectations"],
|
||||
},
|
||||
"actionability": {
|
||||
"weight": 0.15,
|
||||
"mapped_reasons": ["unmet_expectations"],
|
||||
},
|
||||
}
|
||||
|
||||
# Beta(8, 2): prior mean 80/100 with ten equivalent observations. This keeps
|
||||
# one or two votes from producing an extreme public score while real evidence
|
||||
# steadily dominates. With no real sample the API returns ``score=null`` so
|
||||
# the prior is never presented as earned performance.
|
||||
PRIOR_POSITIVE = 8
|
||||
PRIOR_NEGATIVE = 2
|
||||
PRIOR_EQUIVALENT_SAMPLES = PRIOR_POSITIVE + PRIOR_NEGATIVE
|
||||
PRIOR_MEAN = PRIOR_POSITIVE / PRIOR_EQUIVALENT_SAMPLES
|
||||
|
||||
|
||||
def _posterior_score(successes: int, failures: int) -> float | None:
|
||||
evidence_count = successes + failures
|
||||
if evidence_count == 0:
|
||||
return None
|
||||
value = 100.0 * (successes + PRIOR_POSITIVE) / (
|
||||
evidence_count + PRIOR_EQUIVALENT_SAMPLES
|
||||
)
|
||||
return round(value, 1)
|
||||
|
||||
|
||||
def _score_band(score: float | None) -> dict[str, Any]:
|
||||
if score is None:
|
||||
return {
|
||||
"key": "insufficient_data",
|
||||
"label": "数据不足",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
}
|
||||
if score >= 90:
|
||||
return {
|
||||
"key": "excellent",
|
||||
"label": "优秀",
|
||||
"min": 90,
|
||||
"max": 100,
|
||||
}
|
||||
if score >= 80:
|
||||
return {
|
||||
"key": "good",
|
||||
"label": "良好",
|
||||
"min": 80,
|
||||
"max": 89.9,
|
||||
}
|
||||
if score >= 70:
|
||||
return {
|
||||
"key": "needs_attention",
|
||||
"label": "需关注",
|
||||
"min": 70,
|
||||
"max": 79.9,
|
||||
}
|
||||
return {
|
||||
"key": "poor",
|
||||
"label": "较差",
|
||||
"min": 0,
|
||||
"max": 69.9,
|
||||
}
|
||||
|
||||
|
||||
def _confidence_level(sample_count: int) -> str:
|
||||
if sample_count == 0:
|
||||
return "none"
|
||||
if sample_count < 10:
|
||||
return "low"
|
||||
if sample_count < 30:
|
||||
return "medium"
|
||||
return "high"
|
||||
|
||||
|
||||
def calculate_answer_quality_score(
|
||||
rows: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
agent_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Aggregate explicit answer feedback into a transparent 0-100 score."""
|
||||
|
||||
records = list(rows)
|
||||
positive_count = sum(1 for row in records if int(row.get("rating") or 0) == 1)
|
||||
negative_rows = [row for row in records if int(row.get("rating") or 0) == -1]
|
||||
negative_count = len(negative_rows)
|
||||
sample_count = positive_count + negative_count
|
||||
|
||||
reason_counts = {reason: 0 for reason in FEEDBACK_REASONS}
|
||||
negative_reason_sets: list[set[str]] = []
|
||||
for row in negative_rows:
|
||||
raw_reasons = row.get("reasons")
|
||||
if not isinstance(raw_reasons, list):
|
||||
raw_reasons = []
|
||||
reason_set = {
|
||||
str(reason)
|
||||
for reason in raw_reasons
|
||||
if str(reason) in reason_counts
|
||||
}
|
||||
negative_reason_sets.append(reason_set)
|
||||
for reason in reason_set:
|
||||
reason_counts[reason] += 1
|
||||
|
||||
dimensions: dict[str, dict[str, Any]] = {}
|
||||
weighted_total = 0.0
|
||||
available_weight = 0.0
|
||||
for name, config in DIMENSION_CONFIG.items():
|
||||
mapped = set(config["mapped_reasons"])
|
||||
failures = sum(1 for reasons in negative_reason_sets if reasons & mapped)
|
||||
evidence_count = positive_count + failures
|
||||
dimension_score = _posterior_score(positive_count, failures)
|
||||
weight = float(config["weight"])
|
||||
dimensions[name] = {
|
||||
"weight": weight,
|
||||
"score": dimension_score,
|
||||
"evidence_count": evidence_count,
|
||||
"positive_count": positive_count,
|
||||
"negative_count": failures,
|
||||
"mapped_reasons": list(config["mapped_reasons"]),
|
||||
}
|
||||
if dimension_score is not None:
|
||||
weighted_total += weight * dimension_score
|
||||
available_weight += weight
|
||||
|
||||
task_score = (
|
||||
round(weighted_total / available_weight, 1)
|
||||
if available_weight > 0
|
||||
else None
|
||||
)
|
||||
score = _posterior_score(positive_count, negative_count)
|
||||
task_success_rate = (
|
||||
round(positive_count / sample_count, 4) if sample_count else None
|
||||
)
|
||||
reason_breakdown = {
|
||||
reason: {
|
||||
"count": count,
|
||||
"rate": round(count / negative_count, 4) if negative_count else 0.0,
|
||||
}
|
||||
for reason, count in reason_counts.items()
|
||||
}
|
||||
|
||||
return {
|
||||
"agent_id": agent_id,
|
||||
"score": score,
|
||||
"score_band": _score_band(score),
|
||||
"confidence": round(sample_count / (sample_count + 10), 4),
|
||||
"confidence_level": _confidence_level(sample_count),
|
||||
"sample_count": sample_count,
|
||||
"positive_count": positive_count,
|
||||
"negative_count": negative_count,
|
||||
"task_success_rate": task_success_rate,
|
||||
"task_score": task_score,
|
||||
"dimensions": dimensions,
|
||||
"reason_breakdown": reason_breakdown,
|
||||
"prior": {
|
||||
"positive": PRIOR_POSITIVE,
|
||||
"negative": PRIOR_NEGATIVE,
|
||||
"mean": PRIOR_MEAN,
|
||||
"equivalent_samples": PRIOR_EQUIVALENT_SAMPLES,
|
||||
},
|
||||
"formula": {
|
||||
"scope": "user_feedback_answer_quality",
|
||||
"overall": (
|
||||
"100 * (positive_count + 8) / (sample_count + 10); "
|
||||
"null when sample_count is zero"
|
||||
),
|
||||
"confidence": "sample_count / (sample_count + 10)",
|
||||
"dimension": (
|
||||
"100 * (positive_count + 8) / "
|
||||
"(positive_count + mapped_negative_count + 10); "
|
||||
"null when dimension evidence_count is zero"
|
||||
),
|
||||
"task_score": (
|
||||
"weighted mean of evidenced dimensions using weights "
|
||||
"30/20/20/15/15, renormalized over available evidence"
|
||||
),
|
||||
"reason_rate_denominator": "negative_count",
|
||||
"limitations": (
|
||||
"This is not a complete agent-capability score and does not "
|
||||
"grade hidden reasoning, planning, skill/tool use, latency, "
|
||||
"reliability, or safety without separate evidence."
|
||||
),
|
||||
},
|
||||
}
|
||||
+279
-88
@@ -1,17 +1,23 @@
|
||||
"""SQLAlchemy-backed feedback storage.
|
||||
"""SQLAlchemy-backed assistant-answer feedback storage.
|
||||
|
||||
Each method acquires its own short-lived session.
|
||||
Each method acquires its own short-lived session. Writes are idempotent on a
|
||||
SHA-256 target key derived from user/thread/message/agent; the digest avoids
|
||||
oversized compound indexes on legacy MySQL utf8mb4 deployments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy import case, delete, func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.feedback.scoring import FEEDBACK_REASONS, calculate_answer_quality_score
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
|
||||
@@ -20,12 +26,51 @@ class FeedbackRepository:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: FeedbackRow) -> dict:
|
||||
d = row.to_dict()
|
||||
val = d.get("created_at")
|
||||
if isinstance(val, datetime):
|
||||
d["created_at"] = val.isoformat()
|
||||
return d
|
||||
def _row_to_dict(row: FeedbackRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
data["reasons"] = list(data.get("reasons") or [])
|
||||
data["detail"] = data.get("comment")
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def make_target_key(
|
||||
*,
|
||||
user_id: str | None,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
) -> str:
|
||||
"""Build the portable idempotency identity for one answer rating.
|
||||
|
||||
Historical run-level callers have no message id. Their deterministic
|
||||
``run:<id>`` surrogate preserves the old one-rating-per-run behavior.
|
||||
"""
|
||||
|
||||
message_key = message_id or f"run:{run_id}"
|
||||
raw = "\x1f".join(
|
||||
(
|
||||
user_id or "",
|
||||
thread_id,
|
||||
message_key,
|
||||
agent_id or "lead_agent",
|
||||
)
|
||||
)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _normalize_reasons(reasons: list[str] | None) -> list[str]:
|
||||
allowed = set(FEEDBACK_REASONS)
|
||||
normalized: list[str] = []
|
||||
for reason in reasons or []:
|
||||
value = str(reason).strip()
|
||||
if value in allowed and value not in normalized:
|
||||
normalized.append(value)
|
||||
return normalized
|
||||
|
||||
async def create(
|
||||
self,
|
||||
@@ -35,34 +80,29 @@ class FeedbackRepository:
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
message_id: str | None = None,
|
||||
agent_id: str = "lead_agent",
|
||||
reasons: list[str] | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create a feedback record. rating must be +1 or -1."""
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.create")
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
) -> dict[str, Any]:
|
||||
"""Create or update the same answer target idempotently."""
|
||||
|
||||
return await self.upsert(
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
message_id=message_id,
|
||||
rating=rating,
|
||||
user_id=user_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
reasons=reasons,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
feedback_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict | None:
|
||||
) -> dict[str, Any] | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
@@ -72,38 +112,101 @@ class FeedbackRepository:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get_by_target(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any] | None:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.get_by_target",
|
||||
)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = await session.scalar(
|
||||
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
return self._row_to_dict(row) if row is not None else None
|
||||
|
||||
async def list_by_run(
|
||||
self,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
message_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_run")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_run",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
)
|
||||
if message_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.message_id == message_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_by_thread(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
limit: int = 1000,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread")
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_thread",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_by_message_ids(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
message_ids: list[str],
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Load current-user feedback for a bounded visible-message batch."""
|
||||
|
||||
if not message_ids:
|
||||
return []
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_message_ids",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.message_id.in_(message_ids),
|
||||
)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
@@ -111,7 +214,10 @@ class FeedbackRepository:
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete")
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete",
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
if row is None:
|
||||
@@ -129,84 +235,152 @@ class FeedbackRepository:
|
||||
thread_id: str,
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
message_id: str | None = None,
|
||||
agent_id: str = "lead_agent",
|
||||
reasons: list[str] | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create or update feedback for (thread_id, run_id, user_id). rating must be +1 or -1."""
|
||||
) -> dict[str, Any]:
|
||||
"""Create or update feedback for one user/thread/message/agent target."""
|
||||
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.upsert")
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.rating = rating
|
||||
row.comment = comment
|
||||
row.created_at = datetime.now(UTC)
|
||||
else:
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
rating=rating,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.upsert",
|
||||
)
|
||||
normalized_agent_id = str(agent_id or "lead_agent")[:128]
|
||||
normalized_reasons = self._normalize_reasons(reasons)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=normalized_agent_id,
|
||||
)
|
||||
|
||||
# The unique digest handles cross-worker races. On a simultaneous
|
||||
# insert, retry as an update in a fresh transaction.
|
||||
for attempt in range(2):
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
row = await session.scalar(
|
||||
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
if row is None:
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
message_id=message_id,
|
||||
agent_id=normalized_agent_id,
|
||||
target_key=target_key,
|
||||
rating=rating,
|
||||
reasons=normalized_reasons,
|
||||
comment=comment,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.run_id = run_id
|
||||
row.rating = rating
|
||||
row.reasons = normalized_reasons
|
||||
row.comment = comment
|
||||
row.updated_at = now
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
if attempt == 0:
|
||||
continue
|
||||
raise
|
||||
# Runtime factories use expire_on_commit=False. Avoid a
|
||||
# post-commit refresh, unsafe behind a read replica.
|
||||
return self._row_to_dict(row)
|
||||
raise RuntimeError("unreachable feedback upsert retry state")
|
||||
|
||||
async def delete_by_run(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
"""Delete the current user's feedback for a run. Returns True if a record was deleted."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete_by_run")
|
||||
"""Delete this user's feedback for a run or one message in the run."""
|
||||
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete_by_run",
|
||||
)
|
||||
conditions = [
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
]
|
||||
if message_id is not None:
|
||||
conditions.append(FeedbackRow.message_id == message_id)
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
result = await session.execute(delete(FeedbackRow).where(*conditions))
|
||||
await session.commit()
|
||||
return True
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def delete_by_target(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str,
|
||||
agent_id: str,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete_by_target",
|
||||
)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def list_by_thread_grouped(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict[str, dict]:
|
||||
"""Return feedback grouped by run_id for a thread: {run_id: feedback_dict}."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread_grouped")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return {row.run_id: self._row_to_dict(row) for row in result.scalars()}
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Legacy run-keyed view, retaining the newest row for each run."""
|
||||
|
||||
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict:
|
||||
records = await self.list_by_thread(thread_id, user_id=user_id)
|
||||
return {record["run_id"]: record for record in records}
|
||||
|
||||
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict[str, Any]:
|
||||
"""Aggregate feedback stats for a run using database-side counting."""
|
||||
|
||||
stmt = select(
|
||||
func.count().label("total"),
|
||||
func.coalesce(func.sum(case((FeedbackRow.rating == 1, 1), else_=0)), 0).label("positive"),
|
||||
func.coalesce(func.sum(case((FeedbackRow.rating == -1, 1), else_=0)), 0).label("negative"),
|
||||
).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
func.coalesce(
|
||||
func.sum(case((FeedbackRow.rating == 1, 1), else_=0)),
|
||||
0,
|
||||
).label("positive"),
|
||||
func.coalesce(
|
||||
func.sum(case((FeedbackRow.rating == -1, 1), else_=0)),
|
||||
0,
|
||||
).label("negative"),
|
||||
).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).one()
|
||||
return {
|
||||
@@ -215,3 +389,20 @@ class FeedbackRepository:
|
||||
"positive": row.positive,
|
||||
"negative": row.negative,
|
||||
}
|
||||
|
||||
async def aggregate_by_agent(self, agent_id: str) -> dict[str, Any]:
|
||||
"""Return the documented user-feedback answer-quality score."""
|
||||
|
||||
normalized_agent_id = str(agent_id or "lead_agent")[:128]
|
||||
stmt = select(FeedbackRow.rating, FeedbackRow.reasons).where(
|
||||
FeedbackRow.agent_id == normalized_agent_id
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return calculate_answer_quality_score(
|
||||
(
|
||||
{"rating": row.rating, "reasons": list(row.reasons or [])}
|
||||
for row in rows
|
||||
),
|
||||
agent_id=normalized_agent_id,
|
||||
)
|
||||
Reference in new issue
Block a user