cases.json

{
  "as_of": "2026-10-13",
  "notes": "Gold SQL is hand-written independently of the offline planner. 'intent' is what the simulated user picks when asked. 'match_columns' (optional) limits matching to those gold columns.",
  "cases": [
    {"id": "c01", "ambiguous": false, "question": "How many new customers signed up last month?",
     "gold_sql": "SELECT COUNT(*) AS n FROM customers WHERE created_at >= '2026-09-01' AND created_at < '2026-10-01'"},
    {"id": "c02", "ambiguous": false, "question": "How many orders were placed last month?",
     "gold_sql": "SELECT COUNT(*) AS n FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01'"},
    {"id": "c03", "ambiguous": false, "question": "What was our total revenue last month?",
     "gold_sql": "SELECT SUM(order_value) AS revenue FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01'"},
    {"id": "c04", "ambiguous": false, "question": "What was the average order value last month?",
     "gold_sql": "SELECT AVG(order_value) AS aov FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01'"},
    {"id": "c05", "ambiguous": false, "question": "How many orders were canceled last month?",
     "gold_sql": "SELECT COUNT(*) AS n FROM orders WHERE order_status = 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01'"},
    {"id": "c06", "ambiguous": false, "question": "Which payment method brought in the most money last month?",
     "gold_sql": "SELECT pm.payment_type, SUM(pm.payment_value) AS total FROM payments pm JOIN orders od USING (order_id) WHERE od.order_status != 'canceled' AND od.order_date >= '2026-09-01' AND od.order_date < '2026-10-01' GROUP BY pm.payment_type ORDER BY total DESC LIMIT 1"},
    {"id": "c07", "ambiguous": false, "question": "What was revenue by category last month?",
     "gold_sql": "SELECT pr.category, SUM(it.quantity * it.unit_price - it.discount) AS revenue FROM order_items it JOIN orders od USING (order_id) JOIN products pr USING (product_id) WHERE od.order_status != 'canceled' AND od.order_date >= '2026-09-01' AND od.order_date < '2026-10-01' GROUP BY pr.category"},
    {"id": "c08", "ambiguous": false, "question": "How many customers placed at least one order last month?",
     "gold_sql": "SELECT COUNT(*) AS n FROM (SELECT DISTINCT customer_id FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01') x"},
    {"id": "c09", "ambiguous": false, "question": "Show monthly revenue for 2026.", "match_columns": ["revenue"],
     "gold_sql": "SELECT to_char(order_date, 'YYYY-MM') AS month, SUM(order_value) AS revenue FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-01-01' AND order_date < '2027-01-01' GROUP BY 1"},
    {"id": "c10", "ambiguous": false, "question": "How many orders did we get in the last 7 days?",
     "gold_sql": "SELECT COUNT(*) AS n FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-10-06' AND order_date < '2026-10-13'"},
    {"id": "c11", "ambiguous": false, "question": "What is the total number of customers?",
     "gold_sql": "SELECT COUNT(customer_id) AS n FROM customers"},
    {"id": "c12", "ambiguous": false, "question": "Which state has the most customers?",
     "gold_sql": "SELECT customer_state, COUNT(*) AS n FROM customers GROUP BY customer_state ORDER BY n DESC LIMIT 1"},
    {"id": "c13", "ambiguous": false, "question": "What was the revenue in August 2026?",
     "gold_sql": "SELECT SUM(order_value) AS revenue FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-08-01' AND order_date < '2026-09-01'"},
    {"id": "c14", "ambiguous": false, "question": "Top 5 customers by total spend last month",
     "gold_sql": "SELECT customer_id, SUM(order_value) AS spend FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01' GROUP BY customer_id ORDER BY spend DESC LIMIT 5"},
    {"id": "c15", "ambiguous": false, "question": "Which product sold the most units last month?",
     "gold_sql": "SELECT pr.product_name, SUM(it.quantity) AS units FROM order_items it JOIN orders od USING (order_id) JOIN products pr USING (product_id) WHERE od.order_status != 'canceled' AND od.order_date >= '2026-09-01' AND od.order_date < '2026-10-01' GROUP BY pr.product_name ORDER BY units DESC LIMIT 1"},
    {"id": "c16", "ambiguous": false, "question": "Who placed the most orders last month?",
     "gold_sql": "SELECT customer_id, COUNT(*) AS n FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01' GROUP BY customer_id ORDER BY n DESC LIMIT 1"},

    {"id": "a01", "ambiguous": true, "question": "Show me last month's best customer.", "intent": {"best_customer": "revenue"},
     "gold_sql": "SELECT customer_id, SUM(order_value) AS spend FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01' GROUP BY customer_id ORDER BY spend DESC LIMIT 1"},
    {"id": "a02", "ambiguous": true, "question": "Show me last month's best customer.", "intent": {"best_customer": "orders"},
     "gold_sql": "SELECT customer_id, COUNT(*) AS n FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01' GROUP BY customer_id ORDER BY n DESC LIMIT 1"},
    {"id": "a03", "ambiguous": true, "question": "Show me last month's best customer.", "intent": {"best_customer": "frequency"},
     "gold_sql": "SELECT customer_id, COUNT(DISTINCT date_trunc('day', order_date)) AS days FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01' GROUP BY customer_id ORDER BY days DESC LIMIT 1"},
    {"id": "a04", "ambiguous": true, "question": "Who are our top 3 customers this year?", "intent": {"best_customer": "revenue"},
     "gold_sql": "SELECT customer_id, SUM(order_value) AS spend FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-01-01' AND order_date < '2026-10-14' GROUP BY customer_id ORDER BY spend DESC LIMIT 3"},
    {"id": "a05", "ambiguous": true, "question": "What was our best-selling product last month?", "intent": {"best_product": "revenue"},
     "gold_sql": "SELECT pr.product_name, SUM(it.quantity * it.unit_price - it.discount) AS sales FROM order_items it JOIN orders od USING (order_id) JOIN products pr USING (product_id) WHERE od.order_status != 'canceled' AND od.order_date >= '2026-09-01' AND od.order_date < '2026-10-01' GROUP BY pr.product_name ORDER BY sales DESC LIMIT 1"},
    {"id": "a06", "ambiguous": true, "question": "What was the top product last month?", "intent": {"best_product": "units"},
     "gold_sql": "SELECT pr.product_name, SUM(it.quantity) AS units FROM order_items it JOIN orders od USING (order_id) JOIN products pr USING (product_id) WHERE od.order_status != 'canceled' AND od.order_date >= '2026-09-01' AND od.order_date < '2026-10-01' GROUP BY pr.product_name ORDER BY units DESC LIMIT 1"},
    {"id": "a07", "ambiguous": true, "question": "Which category performed best last month?", "intent": {"best_category": "units"},
     "gold_sql": "SELECT pr.category, SUM(it.quantity) AS units FROM order_items it JOIN orders od USING (order_id) JOIN products pr USING (product_id) WHERE od.order_status != 'canceled' AND od.order_date >= '2026-09-01' AND od.order_date < '2026-10-01' GROUP BY pr.category ORDER BY units DESC LIMIT 1"},
    {"id": "a08", "ambiguous": true, "question": "How many new customers did we get last month?", "intent": {"new_customers": "first_order"},
     "gold_sql": "WITH f AS (SELECT customer_id, MIN(order_date) AS d FROM orders WHERE order_status != 'canceled' GROUP BY customer_id) SELECT COUNT(*) AS n FROM f WHERE d >= '2026-09-01' AND d < '2026-10-01'"},
    {"id": "a09", "ambiguous": true, "question": "How many new customers did we get last month?", "intent": {"new_customers": "signup"},
     "gold_sql": "SELECT COUNT(*) AS n FROM customers WHERE created_at >= '2026-09-01' AND created_at < '2026-10-01'"},
    {"id": "a10", "ambiguous": true, "question": "How many orders came in recently?", "intent": {"recent": "last_7_days"},
     "gold_sql": "SELECT COUNT(*) AS n FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-10-06' AND order_date < '2026-10-13'"},
    {"id": "a11", "ambiguous": true, "question": "What's our revenue from recent orders?", "intent": {"recent": "last_30_days"},
     "gold_sql": "SELECT SUM(order_value) AS revenue FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-09-13' AND order_date < '2026-10-13'"},
    {"id": "a12", "ambiguous": true, "question": "How many big orders did we have last month?", "intent": {"large_order": "top_10pct"},
     "gold_sql": "WITH p AS (SELECT percentile_cont(0.9) WITHIN GROUP (ORDER BY order_value) AS v FROM orders WHERE order_status != 'canceled') SELECT COUNT(*) AS n FROM orders, p WHERE order_status != 'canceled' AND order_date >= '2026-09-01' AND order_date < '2026-10-01' AND order_value >= p.v"},
    {"id": "a13", "ambiguous": true, "question": "How many active customers do we have?", "intent": {"active_customers": "active_90"},
     "gold_sql": "SELECT COUNT(DISTINCT customer_id) AS n FROM orders WHERE order_status != 'canceled' AND order_date >= '2026-07-15' AND order_date < '2026-10-13'"},
    {"id": "a14", "ambiguous": true, "question": "Who is our number one customer?", "intent": {"best_customer": "orders"},
     "gold_sql": "SELECT customer_id, COUNT(*) AS n FROM orders WHERE order_status != 'canceled' GROUP BY customer_id ORDER BY n DESC LIMIT 1"}
  ]
}