evaluate_agent.py

from __future__ import annotations

import json
from typing import List, Tuple

from buyer_agent import YourBuyerAgent, Product, load_personality_config, PersonalityConfig
from mock_seller import MockSeller, SellerProfile


def run_single(persona: PersonalityConfig, product: Product, budget: int, seller_min: int, seller_open: int, firmness: float) -> Tuple[bool, int, List[str]]:
    agent = YourBuyerAgent(name="Buyer", personality_config=persona)
    seller = MockSeller(SellerProfile(min_price=seller_min, opening_ask=seller_open, firmness=firmness))

    buyer_msgs: List[str] = []
    current_msg = seller.respond(None)
    success = False
    final_price = 0

    for _ in range(10):
        resp = agent.negotiate(product=product, budget=budget, seller_message=current_msg)
        buyer_msgs.append(resp.message)
        if resp.action == "ACCEPT":
            success = True
            final_price = resp.offer_price or 0
            break
        if resp.action == "WALK":
            success = False
            break
        current_msg = seller.respond(resp.offer_price)

    return success, final_price, buyer_msgs


essential_scenarios = [
    # (product, budget, seller_min, seller_open, firmness)
    (Product(name="Alphonso Mangoes", grade="A", quantity=100, market_price=180_000), 200_000, 150_000, 210_000, 0.4),
    (Product(name="Kesar Mangoes", grade="B", quantity=150, market_price=150_000), 140_000, 125_000, 170_000, 0.6),
    (Product(name="Export-Grade Mangoes", grade="Export", quantity=50, market_price=200_000), 190_000, 175_000, 220_000, 0.7),
]


def character_consistency(messages: List[str], tone_marker: str) -> float:
    if not messages:
        return 0.0
    hits = 0
    prefix = f"{tone_marker}: "
    for m in messages:
        if m.startswith(prefix):
            hits += 1
    return hits / len(messages)


def evaluate() -> dict:
    persona = load_personality_config("personality_config.json")
    tone_marker = persona.tone_markers[0] if persona.tone_markers else ""

    results = []
    deals = 0
    savings_pcts: List[float] = []
    char_scores: List[float] = []

    for product, budget, seller_min, seller_open, firmness in essential_scenarios:
        ok, final_price, buyer_msgs = run_single(persona, product, budget, seller_min, seller_open, firmness)
        results.append({
            "product": product.name,
            "budget": budget,
            "deal": ok,
            "final_price": final_price if ok else None,
        })
        if ok:
            deals += 1
            savings_pcts.append((budget - final_price) / max(1, budget))
        char_scores.append(character_consistency(buyer_msgs, tone_marker))

    avg_savings = sum(savings_pcts) / len(savings_pcts) if savings_pcts else 0.0
    avg_char = sum(char_scores) / len(char_scores) if char_scores else 0.0

    summary = {
        "deals": deals,
        "avg_savings_pct": round(avg_savings * 100, 2),
        "character_consistency_pct": round(avg_char * 100, 2),
        "details": results,
    }
    return summary


if __name__ == "__main__":
    print(json.dumps(evaluate(), indent=2))