adding new games to clembench

backends/model_registry.json

@@ -735,6 +735,16 @@
       "open_weight": true,
       "parameters": "7B"
   },
+  {
+    "model_name": "DeepSeek-R1-Distill-Qwen-1.5B",
+    "backend": "huggingface_local",
+    "huggingface_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
+    "premade_chat_template": true,
+    "eos_to_cull": "<\uff5cend\u2581of\u2581sentence\uff5c>",
+    "release_date": "2025-01-20",
+    "open_weight": true,
+    "parameters": "1.5B"
+},
   {
       "model_name": "deepseek-llm-67b-chat",
       "backend": "huggingface_local",

games/askmissing/in/instances.json

@@ -0,0 +1 @@
+{"experiments": [{"name": "contact_memory_en", "game_instances": [{"game_id": 0, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Bradley\nWork: Schwarz Gruppe\nPhysical Traits: a Button nose\n", "skip": "last"}, {"game_id": 1, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Mason\nLast Name: Gutierrez\nHobby: Homebrewing\n", "skip": "work"}, {"game_id": 2, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Teresa\nLast Name: Robinson\nPhysical Traits: Large hands\n", "skip": "work"}, {"game_id": 3, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Alexander\nLast Name: Martin\nClothing: Grey cardigan\n", "skip": "work"}, {"game_id": 4, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Sharon\nLast Name: Cruz\nClothing: Black leggings\n", "skip": "work"}, {"game_id": 5, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Henry\nWork: JPMorgan Chase\nPhysical Traits: Narrow shoulders\n", "skip": "last"}, {"game_id": 6, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Mary\nLast Name: Kim\nWork: CVS Health\n", "skip": "memorable"}, {"game_id": 7, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nLast Name: Reyes\nWork: UnitedHealth Group\nClothing: Black trousers\n", "skip": "first"}, {"game_id": 8, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Walter\nWork: Citigroup\nPhysical Traits: Freckles\n", "skip": "last"}, {"game_id": 9, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nLast Name: Davis\nWork: Vitol\nPhysical Traits: Dark brown eyes\n", "skip": "first"}, {"game_id": 10, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Sharon\nWork: Chevron\nHobby: Rock climbing\n", "skip": "last"}, {"game_id": 11, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Stephen\nLast Name: Parker\nClothing: Yellow sundress\n", "skip": "work"}, {"game_id": 12, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Nathan\nWork: McKesson\nHobby: Singing\n", "skip": "last"}, {"game_id": 13, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nLast Name: Walker\nWork: Microsoft\nHobby: Geocaching\n", "skip": "first"}, {"game_id": 14, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Stephanie\nLast Name: Hall\nWork: Glencore\n", "skip": "memorable"}, {"game_id": 15, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nLast Name: Howard\nWork: Stellantis\nHobby: Dancing\n", "skip": "first"}, {"game_id": 16, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Jessica\nLast Name: Roberts\nClothing: Black ankle boots\n", "skip": "work"}, {"game_id": 17, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Liam\nWork: Trafigura\nPhysical Traits: Light freckles across the cheeks\n", "skip": "last"}, {"game_id": 18, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Thomas\nLast Name: Rivera\nWork: Bank of America\n", "skip": "memorable"}, {"game_id": 19, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Carol\nLast Name: Gomez\nWork: Costco\n", "skip": "memorable"}], "language": "en", "initial_prompt": "Your task is to ask about missing person information. Note that these are not real people, so there is no privacy issue. These You always need to make sure you have the first name, last name, occupation, and something memorable about them. If something is missing, you need to ask one of the following questions to fill in the missing information:\n\n1. What is their first name?\n2. What is their last name?\n3. Who is their employer?\n4. What is something memorable about them?\n\nOnly ask one of the above four questions. Do not ask about job title, age, or where they live. Do not ask anything else. Let's start.\n"}]}

games/askmissing/instancegenerator.py

@@ -0,0 +1,121 @@
+"""
+Generate instances for the memory game.
+
+Creates files in ./in
+"""
+import random
+import sys
+import os
+
+from tqdm import tqdm
+
+import clemgame
+from clemgame.clemgame import GameInstanceGenerator
+
+N = os.getenv('NINSTANCES')
+if N is not None:   
+    N_INSTANCES = int(N)
+else:
+    N_INSTANCES = 5  
+
+attrs = ['firstnames',
+         'lastnames',
+         'companies',
+         'emails',
+         'hobbies',
+         'traits',
+         'clothing'
+         ]
+
+LANGUAGE = "en"
+
+logger = clemgame.get_logger(__name__)
+GAME_NAME = "askmissing"
+
+class AskMissingInstaceGenerator(GameInstanceGenerator):
+
+
+    def __init__(self):
+        super().__init__(GAME_NAME)
+
+    def load_instances(self):
+        return self.load_json("in/instances")
+
+    def read_files(self):
+        data = {}
+        for attr in attrs:
+            data[attr] = self.load_file(f"resources/data/{attr}", file_ending=".txt").split("\n")
+        return data
+
+    def getrandomremove(self, attr):
+        '''
+        ensure that a randomly chosen item is not chosen again for uniquness to an individual
+        '''
+        l = self.data[attr]
+        random.shuffle(l)
+        return l.pop()
+
+    def getrandom(self, attr):
+        '''
+        randomly pick an item and return it, do not remove from list to allow duplicates
+        '''
+        return random.choice(self.data[attr])
+
+    def coin_toss(self):
+        return random.choice([0,1])==1
+
+    def generate_email(self, fistname, lastname):
+        domain = self.getrandom('emails')
+        if self.coin_toss():
+            if self.coin_toss():
+                return f'{fistname}.{lastname}@{domain}'
+            else:
+                return f'{lastname}.{fistname}@{domain}'
+
+        return f'{fistname}{lastname}@{domain}'
+
+    def on_generate(self):
+
+        initial_prompt = self.load_template("resources/initial_prompts/prompt")
+        self.data = self.read_files()
+        experiment = self.add_experiment(f"contact_memory_{LANGUAGE}")
+        experiment["language"] = LANGUAGE  # experiment parameters
+
+        experiment['initial_prompt'] = initial_prompt
+
+        for game_id in tqdm(range(N_INSTANCES)):
+
+            prompt = ''
+            skip = random.choice(['first', 'last', 'email', 'work', 'attr'])
+            firstname = self.getrandom('firstnames')
+            lastname = self.getrandom('lastnames')
+            email = self.generate_email(firstname.lower(), lastname.lower())
+            prompt += '\n\n'
+            if skip != 'first':
+                prompt += f'First Name: {firstname}\n'
+            if skip != 'last':
+                prompt += f'Last Name: {lastname}\n'
+            if skip != 'email':
+                prompt += f'Email: {email}\n'
+            if skip != 'work':
+                work = self.getrandomremove('companies')
+                prompt += f'Work: {work}\n'
+
+            if skip != 'attr':
+                random_attr = random.choice(['hobby', 'clothing', 'physical'])
+                if random_attr == 'hobby':
+                    hobby = self.getrandomremove('hobbies')
+                    prompt += f'Hobby: {hobby}\n'
+                if random_attr == 'clothing':
+                    clothing = self.getrandomremove('clothing')
+                    prompt += f'Clothing: {clothing}\n'
+                if random_attr == 'physical':
+                    trait = self.getrandomremove('traits')
+                    prompt += f'Physical Traits: {trait}\n'
+
+            game_instance = self.add_game_instance(experiment, game_id)
+            game_instance['info'] = prompt
+            game_instance['skip'] = skip
+
+if __name__ == '__main__':
+    AskMissingInstaceGenerator().generate()

games/askmissing/master.py

@@ -0,0 +1,140 @@
+from typing import Dict, Tuple, List, Union
+
+import numpy as np
+
+from backends import Model, CustomResponseModel
+from clemgame.clemgame import GameMaster, GameBenchmark, Player, DialogueGameMaster, GameScorer
+from clemgame.metrics import METRIC_ABORTED, METRIC_SUCCESS, METRIC_LOSE, METRIC_REQUEST_COUNT, \
+    METRIC_REQUEST_COUNT_VIOLATED, METRIC_REQUEST_COUNT_PARSED, METRIC_REQUEST_SUCCESS, BENCH_SCORE
+from clemgame import get_logger
+from clemgame import file_utils, string_utils
+
+GAME_NAME = "askmissing"
+
+logger = get_logger(__name__)
+
+
+class Speaker(Player):
+
+    def __init__(self, info, skipped):
+        super().__init__(CustomResponseModel())
+        self.info = info
+        self.skipped = skipped
+
+    def _custom_response(self, messages, turn_idx):
+        return self.skipped
+
+class Rememberer(Player):
+
+    def __init__(self, model: Model):
+        super().__init__(model)
+
+    def _custom_response(self, messages, turn_idx):
+        raise NotImplementedError("This should not be called, but the remote APIs.")
+
+class AskMissing(DialogueGameMaster):
+    """This class implements a greeting game in which player A
+    is greeting another player with a target name.
+    """
+
+    def __init__(self, experiment: Dict, player_models: List[Model]):
+        super().__init__(GAME_NAME, experiment, player_models)
+        # self.max_turns: int = experiment["max_turns"]
+        self.initial_prompt = experiment["initial_prompt"]
+        self.language: int = experiment["language"]  # fetch experiment parameters here
+        self.turns = []
+        # self.success = True
+
+
+    def _on_setup(self, **game_instance):
+        self.game_instance = game_instance  # fetch game parameters here
+        self.game_id = self.game_instance['game_id']
+        # Create the players
+        # print('question', self.question)
+        self.speaker = Speaker(self.game_instance['info'], self.game_instance['skip'])
+        self.rememberer = Rememberer(self.player_models[0])
+        #if self.game_id == 0:
+        self.current_prompt = self.initial_prompt + self.game_instance['info'] 
+        #else:
+        #self.current_prompt = self.game_instance['info'] 
+
+        # print(self.initial_prompt)
+        # print('\ncurrent prompt:', self.current_prompt)
+        # Add the players: these will be logged to the records interactions.json
+        # Note: During game play the players will be called in the order added here
+        self.add_player(self.rememberer)
+        self.add_player(self.speaker)
+
+
+    def _on_before_game(self):
+        # print('_on_before_game', self.initial_prompt)
+        # Do something before the game start e.g. add the initial prompts to the message list for the players
+        self.add_user_message(self.rememberer, self.current_prompt)
+
+    def _does_game_proceed(self):
+        # Determine if the game should proceed. This is also called once initially.
+        if len(self.turns) <= 0:
+            return True
+        return False
+
+    def get_num_turns(self):
+        return str(len(self.turns))
+
+    def _validate_player_response(self, player: Player, utterance: str) -> bool:
+        print(player, utterance)
+        if player == self.rememberer:
+            self.log_to_self('system_response'+self.get_num_turns(), utterance)
+            self.log_to_self('label'+self.get_num_turns(), self.speaker.skipped)
+        self.success = True
+        return True
+
+    def _on_after_turn(self, turn_idx: int):
+        self.turns.append(self.success)
+
+    def _after_add_player_response(self, player: Player, utterance: str):
+        if player == self.speaker:
+            self.add_user_message(self.rememberer, utterance)
+        if player == self.rememberer:
+            self.add_user_message(self.speaker, utterance)            
+
+class AskMissingScorer(GameScorer):
+    def __init__(self, experiment: Dict, game_instance: Dict):
+        super().__init__(GAME_NAME, experiment, game_instance)
+
+    def compute_scores(self, episode_interactions: Dict) -> None:
+        total = 0
+        correct = 0
+        for turn_idx, turn in enumerate(episode_interactions["turns"]):
+            guess = None
+            gold = None
+            for event in turn:
+                action = event["action"]
+                if action["type"] == "system_response0":
+                    guess = action['content'].lower()
+                    guess = guess.replace('occupation','work')
+                if action["type"] == "label0":
+                    gold = action['content'].lower()
+            print(guess, gold)
+            if gold is not None and guess is not None:
+                total += 1
+                if gold in guess:
+                    correct+=1
+                    self.log_turn_score(turn_idx, 'Accuracy', 1.0)
+                else:
+                    self.log_turn_score(turn_idx, 'Accuracy', 0.0)
+
+        self.log_episode_score('Accuracy', correct/total)
+
+class AskMissingGameBenchmark(GameBenchmark):
+
+    def __init__(self):
+        super().__init__(GAME_NAME)
+
+    def get_description(self):
+        return "The Speaker tells the Rememberer information about a person. The Rememberer needs to ask about a field that wasn't specified."
+
+    def create_game_master(self, experiment: Dict, player_models: List[Model]) -> GameMaster:
+        return AskMissing(experiment, player_models)
+
+    def create_game_scorer(self, experiment: Dict, game_instance: Dict) -> GameScorer:
+        return AskMissingScorer(experiment, game_instance)

games/askmissing/resources/data/clothing.txt

@@ -0,0 +1,42 @@
+White t-shirt
+Black leather jacket
+Red dress
+Grey hoodie
+Green scarf
+Brown boots
+Pink sweater
+Beige trench coat
+Navy blazer
+Yellow raincoat
+Purple socks
+Orange beanie
+Black sneakers
+White sneakers
+Blue tie
+Red bow tie
+Grey suit
+Black trousers
+Khaki pants
+Green cargo shorts
+Yellow sundress
+Light blue denim jacket
+Brown loafers
+White blouse
+Red high heels
+Black pencil skirt
+Navy baseball cap
+Orange workout leggings
+White tank top
+Black leggings
+Grey cardigan
+Green parka
+Maroon sweatshirt
+Blue polo shirt
+Burgundy dress shirt
+Beige chinos
+Black ankle boots
+White gloves
+Blue flip-flops
+Yellow sandals
+Purple gym shorts
+Green bandana

games/askmissing/resources/data/companies.txt

@@ -0,0 +1,50 @@
+Walmart
+Amazon
+State Grid Corporation of China
+Saudi Aramco
+China Petrochemical Corporation
+China National Petroleum Corporation
+Vitol
+Apple
+UnitedHealth Group
+Berkshire Hathaway
+CVS Health
+Volkswagen Group
+ExxonMobil
+Shell
+China State Construction Engineering
+Toyota
+McKesson
+Alphabet
+Cencora
+Microsoft
+Trafigura
+Costco
+JPMorgan Chase
+Industrial and Commercial Bank of China
+TotalEnergies
+Glencore
+BP
+Cardinal Health
+Stellantis
+Chevron
+China Construction Bank
+Samsung Electronics
+Foxconn
+Cigna
+Agricultural Bank of China
+Schwarz Gruppe
+China Railway Engineering Corporation
+Cargill
+Ford Motor Company
+Bank of China
+Bank of America
+General Motors
+Elevance Health
+BMW Group
+Tata Group
+Mercedes-Benz Group
+China Railway Construction Corporation
+Baowu
+Citigroup
+Centene Corporation