Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 24 additions & 7 deletions data-science/analytics/quality_score.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,19 @@
"""Composite quality score — placeholder.
"""Composite quality score calculation.

TODO: Combine rating, reliability (error rate), usage, and review sentiment
TODO: Output quality score 0-100 per agent
This script reads mock agent usage and rating data, then calculates
a simple quality score for each agent.

Formula:
quality_score = avg_rating * 20
+ normalized_total_runs * 40
+ reliability_score * 40

Where:
- avg_rating rewards agents with higher user ratings.
- normalized_total_runs rewards agents that are used more often.
- reliability_score = 1 - error_rate rewards stable agents with fewer errors.

The script prints the top 5 agents by quality score.
"""

import pandas as pd
Expand All @@ -14,18 +26,23 @@ def compute_quality_scores() -> pd.DataFrame:
usage = pd.read_csv(DATASETS_DIR / "agent_usage.csv")
ratings = pd.read_csv(DATASETS_DIR / "agent_ratings.csv")
merged = usage.merge(ratings, on="agent_id", how="left")

# TODO: Weighted composite formula

if "agent_name" not in merged.columns:
merged["agent_name"] = merged["agent_id"]

merged["quality_score"] = (
merged["avg_rating"] * 20
+ (merged["total_runs"] / merged["total_runs"].max()) * 40
+ (1 - merged["error_rate"]) * 40
).round(1)

return merged[["agent_id", "agent_name", "quality_score"]].sort_values(
"quality_score", ascending=False
return (
merged[["agent_id", "agent_name", "quality_score"]]
.sort_values("quality_score", ascending=False)
.head(5)
)


if __name__ == "__main__":
print("Top 5 Agents by Quality Score:")
print(compute_quality_scores().to_string(index=False))