eval_set.py

"""
eval_set.py — Hand-curated evaluation set for PaderBot.

30 questions covering the user persona (prospective international student),
split roughly:
  - 12 English easy lookups (single-page answer)
  - 6 English multi-hop (needs info from 2+ pages)
  - 8 German equivalents of common queries
  - 4 refusal-cases (out-of-scope or impossible to answer from the corpus)


For each question, fill:
  question: the user's question (verbatim, as they'd type it)
  ground_truth: a 1-3 sentence reference answer that I verified against the corpus
  difficulty: easy | multi_hop | refusal
  language: en | de
"""

EVAL_SET = [
    # ============================================================
    # ENGLISH — EASY LOOKUPS (single-source factual)
    # ============================================================
    {
        "id": "en_easy_1",
        "question": "What is BAföG?",
        "ground_truth": "BAföG is a German state financial aid program for students. It consists of half grant and half interest-free loan, aimed at enabling people to study regardless of their financial situation.",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_2",
        "question": "Where can students live in Paderborn?",
        "ground_truth": "Students can live in residences managed by Studierendenwerk Paderborn — ~1,660 places across multiple complexes (Peter-Hille-Weg, Vogeliusweg, Uhlandstraße, etc.). Private rentals are also available through accommodation portals.",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_3",
        "question": "What focus areas does the Computer Science Master's program have?",
        "ground_truth": "The CS Master at Paderborn has five focus areas: Classical and Quantum Algorithm Design; Computer and Communication Systems; Data Science and Intelligent Systems; Security; Software Engineering.",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_4",
        "question": "Is the Computer Science Master's taught in English?",
        "ground_truth": "Yes, the Computer Science Master's program is taught in English.",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_5",
        "question": "What is the minimum GPA required for the Computer Science Master's?",
        "ground_truth": "A final grade of 3.0 or better (on the German scale, where 1.0 is best) in a Bachelor's degree in Computer Science or an equivalent.",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_6",
        "question": "Do international students need health insurance?",
        "ground_truth": "Yes, as an international student, you need health insurance. There are two types of health insurance: statutory health insurance and private health insurance. If your age is above 30 years, the rules have changed.",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_7",
        "question": "What is Uni-Assist?",
        "ground_truth": "Uni-Assist is a portal. As an international Applicant, you use it to apply, which allows you to submit your application electronically. ",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_8",
        "question": "What does the Mensa offer?",
        "ground_truth": "Mensa is a place where you can eat and drink on campus. There are multiple places to eat and drink, Mensa Academica and the Mensa Forum, the Grill|Café and Café Bona Vista. ",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_9",
        "question": "How many places are available in student residences in Paderborn?",
        "ground_truth": "Around 1,660 places at Paderborn University, with additional options in Hamm and Lippstadt.",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_10",
        "question": "What is the International Office?",
        "ground_truth": "International Office is a place which clear all the doubts of studying abroad (application deadlines and procedures, internships, scholarships). ",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_11",
        "question": "Is the Computer Engineering Master's in English?",
        "ground_truth": "yes, Computer Engineering Master's is taught in English.",
        "difficulty": "easy",
        "language": "en",
    },
    {
        "id": "en_easy_12",
        "question": "What is the Electrical Systems Engineering Master's about?",
        "ground_truth": "Electrical System Engineering Master's focus on electrical systems at an advanced level. As a student, it gives you insights into current trends and developments, in addition to ample opportunities for practical experience",
        "difficulty": "easy",
        "language": "en",
    },

    # ============================================================
    # ENGLISH — MULTI-HOP (needs info from 2+ pages/chunks)
    # ============================================================
    {
        "id": "en_multi_1",
        "question": "What are my financing options as an international student at Paderborn?",
        "ground_truth": "International students at Paderborn can finance their studies via BAföG (income-dependent state aid, half grant + half interest-free loan), scholarships, student loans, education funds, or part-time work.",
        "difficulty": "multi_hop",
        "language": "en",
    },
    {
        "id": "en_multi_2",
        "question": "How do I apply to the Computer Engineering Master's program and what documents do I need?",
        "ground_truth": "International applicants apply via the Uni-Assist online portal. Documents typically include transcripts, a Bachelor's certificate (CS/EE/related), English language proof, and a CV. Contact the International Office before registering.",
        "difficulty": "multi_hop",
        "language": "en",
    },
    {
        "id": "en_multi_3",
        "question": "What can I expect for housing costs and how do I apply?",
        "ground_truth": "Expected Housing costs vary depending on the type and size of the accommodation. Generally, it costs 340-450 euros per month. And you can apply for housing through an online application via the accommodation application form. Waiting time goes 1-2 semesters. ",
        "difficulty": "multi_hop",
        "language": "en",
    },
    {
        "id": "en_multi_4",
        "question": "Do I need German skills to study CS, and what programs are taught in English?",
        "ground_truth": "No, you don't need German skills to study CS in Paderborn University, because the CS Program language of teaching is English. There are other English-taught programs in Paderborn University, Computer Engineering Master's, International Economics and Management Master's program, but German skills requirements vary program to program.",
        "difficulty": "multi_hop",
        "language": "en",
    },
    {
        "id": "en_multi_5",
        "question": "What support is available for international students after arrival?",
        "ground_truth": "International Office helps international students with information on enrollment process, changing degree program and other services that international students require on arrival. The university offers free German courses to help you learn the language to adapt to Germany and Job fairs to help you build Connections.  ",
        "difficulty": "multi_hop",
        "language": "en",
    },
    {
        "id": "en_multi_6",
        "question": "What master's programs are taught in English and what are their admission requirements?",
        "ground_truth": "The master's programs that are taught in English are Master's in Computer Science, Master's in Electrical System Engineering, Master's in Computer Engineering and their admission requirements. You will get details from the International Office Page, and as an International Applicant, you need to apply through Uni- Assist.",
        "difficulty": "multi_hop",
        "language": "en",
    },

    # ============================================================
    # GERMAN — EQUIVALENTS OF COMMON QUERIES
    # ============================================================
    {
        "id": "de_1",
        "question": "Was ist BAföG?",
        "ground_truth": "BAföG ist das Bundesausbildungsförderungsgesetz — eine staatliche Förderung für Studierende, die zur Hälfte als Zuschuss und zur Hälfte als zinsloses Darlehen vergeben wird.",
        "difficulty": "easy",
        "language": "de",
    },
    {
        "id": "de_2",
        "question": "Wo kann ich in Paderborn wohnen?",
        "ground_truth": "Das Studierendenwerk Paderborn betreibt mehrere Studierendenwohnheime in Paderborn (u.a. Peter-Hille-Weg, Vogeliusweg, Uhlandstraße) sowie weitere Standorte in Hamm und Lippstadt — insgesamt etwa 1.660 Wohnplätze.",
        "difficulty": "easy",
        "language": "de",
    },
    {
        "id": "de_3",
        "question": "Wie bewerbe ich mich für den Computer Science Master?",
        "ground_truth": "Um sich für einen Masterstudiengang in Informatik an der Universität Paderborn zu bewerben, müssen Sie gute Abschlussnoten vorweisen – gemäß der deutschen Notenskala einen Durchschnitt von 2,0 oder besser. Je nach den geltenden Anforderungen kann zudem die Ablegung des GRE-Tests erforderlich sein. Als internationaler Bewerber müssen Sie Ihre Bewerbung über Uni-Assist einreichen; bitte informieren Sie sich vor der Bewerbung auf der Webseite des International Office, um alle notwendigen Informationen zum Zulassungsverfahren zu erhalten.",
        "difficulty": "multi_hop",
        "language": "de",
    },
    {
        "id": "de_4",
        "question": "Welche Focus Areas hat der Computer Science Master?",
        "ground_truth": "Fünf Focus Areas: Classical and Quantum Algorithm Design; Computer and Communication Systems; Data Science and Intelligent Systems; Security; Software Engineering.",
        "difficulty": "easy",
        "language": "de",
    },
    {
        "id": "de_5",
        "question": "Welche Voraussetzungen brauche ich für den Master in Computer Engineering?",
        "ground_truth": "Um sich für einen Masterstudiengang in Computer Engineering in Paderborn zu bewerben, benötigen Sie einen Bachelorabschluss in Computer Engineering oder einen gleichwertigen Abschluss. Zudem sind Englischkenntnisse auf dem Niveau B2 des Gemeinsamen Europäischen Referenzrahmens für Sprachen (GER) erforderlich. Als internationaler Studierender aus einem Nicht-EU-Land müssen Sie den GRE-Test ablegen. Sollten Sie jedoch über einen hervorragenden Bachelorabschluss verfügen, kann dieser Test unter Umständen entfallen. Für weitere Informationen wenden Sie sich bitte an das International Office.",
        "difficulty": "easy",
        "language": "de",
    },
    {
        "id": "de_6",
        "question": "Wie finanziere ich mein Studium?",
        "ground_truth": "Es gibt verschiedene Möglichkeiten, das Studium zu finanzieren; so bietet beispielsweise das BAföG Beratung zur finanziellen Unterstützung während des Studiums an. Darüber hinaus existieren weitere Optionen, wie etwa Stipendien, Bildungsfonds, Darlehen, Nebenjobs oder Förderprogramme.",
        "difficulty": "multi_hop",
        "language": "de",
    },
    {
        "id": "de_7",
        "question": "Was kostet ein Wohnplatz im Studierendenwohnheim?",
        "ground_truth": "Die Kosten für ein Zimmer in einem Studentenwohnheim variieren je nach Zimmergröße – von 340 Euro für ein Einzelapartment mit 13,50 m² bis hin zu 585,00 Euro für ein Studio-Apartment mit 27–33 m², das an zwei Personen vermietet wird. Für weitere Einzelheiten können Sie die Website des Studierendenwerks Paderborn besuchen oder dieses direkt kontaktieren (während der Geschäftszeiten).",
        "difficulty": "easy",
        "language": "de",
    },
    {
        "id": "de_8",
        "question": "Gibt es englischsprachige Masterprogramme?",
        "ground_truth": "Ja, an der Universität Paderborn gibt es englischsprachige Masterstudiengänge. Dazu gehören der Masterstudiengang Informatik, der Master in Computer Engineering, der Master in International Economics and Management sowie weitere Programme. Für einige dieser Studiengänge sind keinerlei Deutschkenntnisse erforderlich, während andere grundlegende Deutschkenntnisse auf dem Niveau A2 voraussetzen.",
        "difficulty": "easy",
        "language": "de",
    },

    # ============================================================
    # REFUSAL CASES (out of scope or impossible from corpus)
    # ============================================================
    {
        "id": "refusal_1",
        "question": "Who is the president of the United States?",
        "ground_truth": "I don't have enough information to answer that based on the available sources.",
        "difficulty": "refusal",
        "language": "en",
    },
    {
        "id": "refusal_2",
        "question": "What is the weather in Paderborn today?",
        "ground_truth": "I don't have enough information to answer that based on the available sources.",
        "difficulty": "refusal",
        "language": "en",
    },
    {
        "id": "refusal_3",
        "question": "Wie spät ist es jetzt?",
        "ground_truth": "Ich habe nicht genug Informationen, um diese Frage anhand der verfügbaren Quellen zu beantworten.",
        "difficulty": "refusal",
        "language": "de",
    },
    {
        "id": "refusal_4",
        "question": "What's the best restaurant in Berlin?",
        "ground_truth": "I don't have enough information to answer that based on the available sources.",
        "difficulty": "refusal",
        "language": "en",
    },
]


if __name__ == "__main__":
    from collections import Counter
    print(f"Total questions: {len(EVAL_SET)}")
    print(f"By difficulty: {Counter(q['difficulty'] for q in EVAL_SET)}")
    print(f"By language:   {Counter(q['language'] for q in EVAL_SET)}")
    fillins = sum(1 for q in EVAL_SET if "[FILL IN" in q["ground_truth"])
    print(f"\n[FILL IN] placeholders still to fill: {fillins}")
    if fillins:
        print("→ Open eval_set.py and replace [FILL IN] entries with verified answers.")