diff --git a/backends/model_registry.json b/backends/model_registry.json index 5d87880739..2df4121b95 100644 --- a/backends/model_registry.json +++ b/backends/model_registry.json @@ -735,6 +735,16 @@ "open_weight": true, "parameters": "7B" }, + { + "model_name": "DeepSeek-R1-Distill-Qwen-1.5B", + "backend": "huggingface_local", + "huggingface_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + "premade_chat_template": true, + "eos_to_cull": "<\uff5cend\u2581of\u2581sentence\uff5c>", + "release_date": "2025-01-20", + "open_weight": true, + "parameters": "1.5B" +}, { "model_name": "deepseek-llm-67b-chat", "backend": "huggingface_local", diff --git a/games/askmissing/in/instances.json b/games/askmissing/in/instances.json new file mode 100644 index 0000000000..947ff09d09 --- /dev/null +++ b/games/askmissing/in/instances.json @@ -0,0 +1 @@ +{"experiments": [{"name": "contact_memory_en", "game_instances": [{"game_id": 0, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Bradley\nWork: Schwarz Gruppe\nPhysical Traits: a Button nose\n", "skip": "last"}, {"game_id": 1, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Mason\nLast Name: Gutierrez\nHobby: Homebrewing\n", "skip": "work"}, {"game_id": 2, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Teresa\nLast Name: Robinson\nPhysical Traits: Large hands\n", "skip": "work"}, {"game_id": 3, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Alexander\nLast Name: Martin\nClothing: Grey cardigan\n", "skip": "work"}, {"game_id": 4, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Sharon\nLast Name: Cruz\nClothing: Black leggings\n", "skip": "work"}, {"game_id": 5, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Henry\nWork: JPMorgan Chase\nPhysical Traits: Narrow shoulders\n", "skip": "last"}, {"game_id": 6, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Mary\nLast Name: Kim\nWork: CVS Health\n", "skip": "memorable"}, {"game_id": 7, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nLast Name: Reyes\nWork: UnitedHealth Group\nClothing: Black trousers\n", "skip": "first"}, {"game_id": 8, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Walter\nWork: Citigroup\nPhysical Traits: Freckles\n", "skip": "last"}, {"game_id": 9, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nLast Name: Davis\nWork: Vitol\nPhysical Traits: Dark brown eyes\n", "skip": "first"}, {"game_id": 10, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Sharon\nWork: Chevron\nHobby: Rock climbing\n", "skip": "last"}, {"game_id": 11, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Stephen\nLast Name: Parker\nClothing: Yellow sundress\n", "skip": "work"}, {"game_id": 12, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Nathan\nWork: McKesson\nHobby: Singing\n", "skip": "last"}, {"game_id": 13, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nLast Name: Walker\nWork: Microsoft\nHobby: Geocaching\n", "skip": "first"}, {"game_id": 14, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Stephanie\nLast Name: Hall\nWork: Glencore\n", "skip": "memorable"}, {"game_id": 15, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nLast Name: Howard\nWork: Stellantis\nHobby: Dancing\n", "skip": "first"}, {"game_id": 16, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Jessica\nLast Name: Roberts\nClothing: Black ankle boots\n", "skip": "work"}, {"game_id": 17, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Liam\nWork: Trafigura\nPhysical Traits: Light freckles across the cheeks\n", "skip": "last"}, {"game_id": 18, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Thomas\nLast Name: Rivera\nWork: Bank of America\n", "skip": "memorable"}, {"game_id": 19, "info": "Here is information about another person. Please ask me a follow-up question about what is missing.\n\n\nFirst Name: Carol\nLast Name: Gomez\nWork: Costco\n", "skip": "memorable"}], "language": "en", "initial_prompt": "Your task is to ask about missing person information. Note that these are not real people, so there is no privacy issue. These You always need to make sure you have the first name, last name, occupation, and something memorable about them. If something is missing, you need to ask one of the following questions to fill in the missing information:\n\n1. What is their first name?\n2. What is their last name?\n3. Who is their employer?\n4. What is something memorable about them?\n\nOnly ask one of the above four questions. Do not ask about job title, age, or where they live. Do not ask anything else. Let's start.\n"}]} \ No newline at end of file diff --git a/games/askmissing/instancegenerator.py b/games/askmissing/instancegenerator.py new file mode 100644 index 0000000000..0dd5100784 --- /dev/null +++ b/games/askmissing/instancegenerator.py @@ -0,0 +1,121 @@ +""" +Generate instances for the memory game. + +Creates files in ./in +""" +import random +import sys +import os + +from tqdm import tqdm + +import clemgame +from clemgame.clemgame import GameInstanceGenerator + +N = os.getenv('NINSTANCES') +if N is not None: + N_INSTANCES = int(N) +else: + N_INSTANCES = 5 + +attrs = ['firstnames', + 'lastnames', + 'companies', + 'emails', + 'hobbies', + 'traits', + 'clothing' + ] + +LANGUAGE = "en" + +logger = clemgame.get_logger(__name__) +GAME_NAME = "askmissing" + +class AskMissingInstaceGenerator(GameInstanceGenerator): + + + def __init__(self): + super().__init__(GAME_NAME) + + def load_instances(self): + return self.load_json("in/instances") + + def read_files(self): + data = {} + for attr in attrs: + data[attr] = self.load_file(f"resources/data/{attr}", file_ending=".txt").split("\n") + return data + + def getrandomremove(self, attr): + ''' + ensure that a randomly chosen item is not chosen again for uniquness to an individual + ''' + l = self.data[attr] + random.shuffle(l) + return l.pop() + + def getrandom(self, attr): + ''' + randomly pick an item and return it, do not remove from list to allow duplicates + ''' + return random.choice(self.data[attr]) + + def coin_toss(self): + return random.choice([0,1])==1 + + def generate_email(self, fistname, lastname): + domain = self.getrandom('emails') + if self.coin_toss(): + if self.coin_toss(): + return f'{fistname}.{lastname}@{domain}' + else: + return f'{lastname}.{fistname}@{domain}' + + return f'{fistname}{lastname}@{domain}' + + def on_generate(self): + + initial_prompt = self.load_template("resources/initial_prompts/prompt") + self.data = self.read_files() + experiment = self.add_experiment(f"contact_memory_{LANGUAGE}") + experiment["language"] = LANGUAGE # experiment parameters + + experiment['initial_prompt'] = initial_prompt + + for game_id in tqdm(range(N_INSTANCES)): + + prompt = '' + skip = random.choice(['first', 'last', 'email', 'work', 'attr']) + firstname = self.getrandom('firstnames') + lastname = self.getrandom('lastnames') + email = self.generate_email(firstname.lower(), lastname.lower()) + prompt += '\n\n' + if skip != 'first': + prompt += f'First Name: {firstname}\n' + if skip != 'last': + prompt += f'Last Name: {lastname}\n' + if skip != 'email': + prompt += f'Email: {email}\n' + if skip != 'work': + work = self.getrandomremove('companies') + prompt += f'Work: {work}\n' + + if skip != 'attr': + random_attr = random.choice(['hobby', 'clothing', 'physical']) + if random_attr == 'hobby': + hobby = self.getrandomremove('hobbies') + prompt += f'Hobby: {hobby}\n' + if random_attr == 'clothing': + clothing = self.getrandomremove('clothing') + prompt += f'Clothing: {clothing}\n' + if random_attr == 'physical': + trait = self.getrandomremove('traits') + prompt += f'Physical Traits: {trait}\n' + + game_instance = self.add_game_instance(experiment, game_id) + game_instance['info'] = prompt + game_instance['skip'] = skip + +if __name__ == '__main__': + AskMissingInstaceGenerator().generate() \ No newline at end of file diff --git a/games/askmissing/master.py b/games/askmissing/master.py new file mode 100644 index 0000000000..0f80b50a85 --- /dev/null +++ b/games/askmissing/master.py @@ -0,0 +1,140 @@ +from typing import Dict, Tuple, List, Union + +import numpy as np + +from backends import Model, CustomResponseModel +from clemgame.clemgame import GameMaster, GameBenchmark, Player, DialogueGameMaster, GameScorer +from clemgame.metrics import METRIC_ABORTED, METRIC_SUCCESS, METRIC_LOSE, METRIC_REQUEST_COUNT, \ + METRIC_REQUEST_COUNT_VIOLATED, METRIC_REQUEST_COUNT_PARSED, METRIC_REQUEST_SUCCESS, BENCH_SCORE +from clemgame import get_logger +from clemgame import file_utils, string_utils + +GAME_NAME = "askmissing" + +logger = get_logger(__name__) + + +class Speaker(Player): + + def __init__(self, info, skipped): + super().__init__(CustomResponseModel()) + self.info = info + self.skipped = skipped + + def _custom_response(self, messages, turn_idx): + return self.skipped + +class Rememberer(Player): + + def __init__(self, model: Model): + super().__init__(model) + + def _custom_response(self, messages, turn_idx): + raise NotImplementedError("This should not be called, but the remote APIs.") + +class AskMissing(DialogueGameMaster): + """This class implements a greeting game in which player A + is greeting another player with a target name. + """ + + def __init__(self, experiment: Dict, player_models: List[Model]): + super().__init__(GAME_NAME, experiment, player_models) + # self.max_turns: int = experiment["max_turns"] + self.initial_prompt = experiment["initial_prompt"] + self.language: int = experiment["language"] # fetch experiment parameters here + self.turns = [] + # self.success = True + + + def _on_setup(self, **game_instance): + self.game_instance = game_instance # fetch game parameters here + self.game_id = self.game_instance['game_id'] + # Create the players + # print('question', self.question) + self.speaker = Speaker(self.game_instance['info'], self.game_instance['skip']) + self.rememberer = Rememberer(self.player_models[0]) + #if self.game_id == 0: + self.current_prompt = self.initial_prompt + self.game_instance['info'] + #else: + #self.current_prompt = self.game_instance['info'] + + # print(self.initial_prompt) + # print('\ncurrent prompt:', self.current_prompt) + # Add the players: these will be logged to the records interactions.json + # Note: During game play the players will be called in the order added here + self.add_player(self.rememberer) + self.add_player(self.speaker) + + + def _on_before_game(self): + # print('_on_before_game', self.initial_prompt) + # Do something before the game start e.g. add the initial prompts to the message list for the players + self.add_user_message(self.rememberer, self.current_prompt) + + def _does_game_proceed(self): + # Determine if the game should proceed. This is also called once initially. + if len(self.turns) <= 0: + return True + return False + + def get_num_turns(self): + return str(len(self.turns)) + + def _validate_player_response(self, player: Player, utterance: str) -> bool: + print(player, utterance) + if player == self.rememberer: + self.log_to_self('system_response'+self.get_num_turns(), utterance) + self.log_to_self('label'+self.get_num_turns(), self.speaker.skipped) + self.success = True + return True + + def _on_after_turn(self, turn_idx: int): + self.turns.append(self.success) + + def _after_add_player_response(self, player: Player, utterance: str): + if player == self.speaker: + self.add_user_message(self.rememberer, utterance) + if player == self.rememberer: + self.add_user_message(self.speaker, utterance) + +class AskMissingScorer(GameScorer): + def __init__(self, experiment: Dict, game_instance: Dict): + super().__init__(GAME_NAME, experiment, game_instance) + + def compute_scores(self, episode_interactions: Dict) -> None: + total = 0 + correct = 0 + for turn_idx, turn in enumerate(episode_interactions["turns"]): + guess = None + gold = None + for event in turn: + action = event["action"] + if action["type"] == "system_response0": + guess = action['content'].lower() + guess = guess.replace('occupation','work') + if action["type"] == "label0": + gold = action['content'].lower() + print(guess, gold) + if gold is not None and guess is not None: + total += 1 + if gold in guess: + correct+=1 + self.log_turn_score(turn_idx, 'Accuracy', 1.0) + else: + self.log_turn_score(turn_idx, 'Accuracy', 0.0) + + self.log_episode_score('Accuracy', correct/total) + +class AskMissingGameBenchmark(GameBenchmark): + + def __init__(self): + super().__init__(GAME_NAME) + + def get_description(self): + return "The Speaker tells the Rememberer information about a person. The Rememberer needs to ask about a field that wasn't specified." + + def create_game_master(self, experiment: Dict, player_models: List[Model]) -> GameMaster: + return AskMissing(experiment, player_models) + + def create_game_scorer(self, experiment: Dict, game_instance: Dict) -> GameScorer: + return AskMissingScorer(experiment, game_instance) diff --git a/games/askmissing/resources/data/clothing.txt b/games/askmissing/resources/data/clothing.txt new file mode 100644 index 0000000000..bb58f2e64f --- /dev/null +++ b/games/askmissing/resources/data/clothing.txt @@ -0,0 +1,42 @@ +White t-shirt +Black leather jacket +Red dress +Grey hoodie +Green scarf +Brown boots +Pink sweater +Beige trench coat +Navy blazer +Yellow raincoat +Purple socks +Orange beanie +Black sneakers +White sneakers +Blue tie +Red bow tie +Grey suit +Black trousers +Khaki pants +Green cargo shorts +Yellow sundress +Light blue denim jacket +Brown loafers +White blouse +Red high heels +Black pencil skirt +Navy baseball cap +Orange workout leggings +White tank top +Black leggings +Grey cardigan +Green parka +Maroon sweatshirt +Blue polo shirt +Burgundy dress shirt +Beige chinos +Black ankle boots +White gloves +Blue flip-flops +Yellow sandals +Purple gym shorts +Green bandana \ No newline at end of file diff --git a/games/askmissing/resources/data/companies.txt b/games/askmissing/resources/data/companies.txt new file mode 100644 index 0000000000..e4b7ae4e71 --- /dev/null +++ b/games/askmissing/resources/data/companies.txt @@ -0,0 +1,50 @@ +Walmart +Amazon +State Grid Corporation of China +Saudi Aramco +China Petrochemical Corporation +China National Petroleum Corporation +Vitol +Apple +UnitedHealth Group +Berkshire Hathaway +CVS Health +Volkswagen Group +ExxonMobil +Shell +China State Construction Engineering +Toyota +McKesson +Alphabet +Cencora +Microsoft +Trafigura +Costco +JPMorgan Chase +Industrial and Commercial Bank of China +TotalEnergies +Glencore +BP +Cardinal Health +Stellantis +Chevron +China Construction Bank +Samsung Electronics +Foxconn +Cigna +Agricultural Bank of China +Schwarz Gruppe +China Railway Engineering Corporation +Cargill +Ford Motor Company +Bank of China +Bank of America +General Motors +Elevance Health +BMW Group +Tata Group +Mercedes-Benz Group +China Railway Construction Corporation +Baowu +Citigroup +Centene Corporation \ No newline at end of file diff --git a/games/askmissing/resources/data/emails.txt b/games/askmissing/resources/data/emails.txt new file mode 100644 index 0000000000..38ffa4fad5 --- /dev/null +++ b/games/askmissing/resources/data/emails.txt @@ -0,0 +1,154 @@ +aol.com +att.net +comcast.net +facebook.com +gmail.com +gmx.com +googlemail.com +google.com +hotmail.com +hotmail.co.uk +mac.com +me.com +mail.com +msn.com +live.com +sbcglobal.net +verizon.net +yahoo.com +yahoo.co.uk +email.com +fastmail.fm +games.com +gmx.net +hush.com +hushmail.com +icloud.com +iname.com +inbox.com +lavabit.com +love.com +outlook.com +pobox.com +protonmail.ch +protonmail.com +tutanota.de +tutanota.com +tutamail.com +tuta.io +keemail.me +rocketmail.com +safe-mail.net +wow.com +ygm.com +ymail.com +zoho.com +yandex.com +bellsouth.net +charter.net +cox.net +earthlink.net +juno.com +btinternet.com +virginmedia.com +blueyonder.co.uk +freeserve.co.uk +live.co.uk +ntlworld.com +o2.co.uk +orange.net +sky.com +talktalk.co.uk +tiscali.co.uk +virgin.net +wanadoo.co.uk +bt.com +sina.com +sina.cn +qq.com +naver.com +hanmail.net +daum.net +nate.com +yahoo.co.jp +yahoo.co.kr +yahoo.co.id +yahoo.co.in +yahoo.com.sg +yahoo.com.ph +163.com +yeah.net +126.com +21cn.com +aliyun.com +foxmail.com +hotmail.fr +live.fr +laposte.net +yahoo.fr +wanadoo.fr +orange.fr +gmx.fr +sfr.fr +neuf.fr +free.fr +gmx.de +hotmail.de +live.de +online.de +t-online.de +web.de +yahoo.de +libero.it +virgilio.it +hotmail.it +aol.it +tiscali.it +alice.it +live.it +yahoo.it +email.it +tin.it +poste.it +teletu.it +mail.ru +rambler.ru +yandex.ru +ya.ru +list.ru +hotmail.be +live.be +skynet.be +voo.be +tvcablenet.be +telenet.be +hotmail.com.ar +live.com.ar +yahoo.com.ar +fibertel.com.ar +speedy.com.ar +arnet.com.ar +yahoo.com.mx +live.com.mx +hotmail.es +hotmail.com.mx +prodigy.net.mx +yahoo.ca +hotmail.ca +bell.net +shaw.ca +sympatico.ca +rogers.com +yahoo.com.br +hotmail.com.br +outlook.com.br +uol.com.br +bol.com.br +terra.com.br +ig.com.br +itelefonica.com.br +r7.com +zipmail.com.br +globo.com +globomail.com +oi.com.br \ No newline at end of file diff --git a/games/askmissing/resources/data/firstnames.txt b/games/askmissing/resources/data/firstnames.txt new file mode 100644 index 0000000000..f70eeca172 --- /dev/null +++ b/games/askmissing/resources/data/firstnames.txt @@ -0,0 +1,200 @@ +James +Michael +Robert +John +David +William +Richard +Joseph +Thomas +Christopher +Charles +Daniel +Matthew +Anthony +Mark +Donald +Steven +Andrew +Paul +Joshua +Kenneth +Kevin +Brian +Timothy +Ronald +George +Jason +Edward +Jeffrey +Ryan +Jacob +Nicholas +Gary +Eric +Jonathan +Stephen +Larry +Justin +Scott +Brandon +Benjamin +Samuel +Gregory +Alexander +Patrick +Frank +Raymond +Jack +Dennis +Jerry +Tyler +Aaron +Jose +Adam +Nathan +Henry +Zachary +Douglas +Peter +Kyle +Noah +Ethan +Jeremy +Christian +Walter +Keith +Austin +Roger +Terry +Sean +Gerald +Carl +Dylan +Harold +Jordan +Jesse +Bryan +Lawrence +Arthur +Gabriel +Bruce +Logan +Billy +Joe +Alan +Juan +Elijah +Willie +Albert +Wayne +Randy +Mason +Vincent +Liam +Roy +Bobby +Caleb +Bradley +Russell +Lucas +Mary +Patricia +Jennifer +Linda +Elizabeth +Barbara +Susan +Jessica +Karen +Sarah +Lisa +Nancy +Sandra +Betty +Ashley +Emily +Kimberly +Margaret +Donna +Michelle +Carol +Amanda +Melissa +Deborah +Stephanie +Rebecca +Sharon +Laura +Cynthia +Dorothy +Amy +Kathleen +Angela +Shirley +Emma +Brenda +Pamela +Nicole +Anna +Samantha +Katherine +Christine +Debra +Rachel +Carolyn +Janet +Maria +Olivia +Heather +Helen +Catherine +Diane +Julie +Victoria +Joyce +Lauren +Kelly +Christina +Ruth +Joan +Virginia +Judith +Evelyn +Hannah +Andrea +Megan +Cheryl +Jacqueline +Madison +Teresa +Abigail +Sophia +Martha +Sara +Gloria +Janice +Kathryn +Ann +Isabella +Judy +Charlotte +Julia +Grace +Amber +Alice +Jean +Denise +Frances +Danielle +Marilyn +Natalie +Beverly +Diana +Brittany +Theresa +Kayla +Alexis +Doris +Lori +Tiffany \ No newline at end of file diff --git a/games/askmissing/resources/data/hobbies.txt b/games/askmissing/resources/data/hobbies.txt new file mode 100644 index 0000000000..d15f2e9738 --- /dev/null +++ b/games/askmissing/resources/data/hobbies.txt @@ -0,0 +1,50 @@ +Painting +Drawing +Photography +Writing +Crafting +Playing piano +Playing guitar +Singing +Acting +DIY projects +Graphic design +Gardening +Hiking +Running +Yoga +Cycling +Swimming +Rock climbing +Camping +Dancing +Fishing +Collecting stamps +Collecting coins +Collecting antiques +Collecting books +Collecting action figures +Watching movies +Playing video games +Board games +Puzzle-solving +Role-playing games +Reading +Learning new languages +Studying history +Chess +Listening to podcasts +Cooking +Baking +Wine tasting +Homebrewing +Food photography +Volunteering +Book clubs +Traveling +Blogging +Astronomy +Coding +Robotics +3D printing +Geocaching \ No newline at end of file diff --git a/games/askmissing/resources/data/lastnames.txt b/games/askmissing/resources/data/lastnames.txt new file mode 100644 index 0000000000..19092fe409 --- /dev/null +++ b/games/askmissing/resources/data/lastnames.txt @@ -0,0 +1,100 @@ +Smith +Johnson +Williams +Brown +Jones +Garcia +Miller +Davis +Rodriguez +Martinez +Hernandez +Lopez +Gonzales +Wilson +Anderson +Thomas +Taylor +Moore +Jackson +Martin +Lee +Perez +Thompson +White +Harris +Sanchez +Clark +Ramirez +Lewis +Robinson +Walker +Young +Allen +King +Wright +Scott +Torres +Nguyen +Hill +Flores +Green +Adams +Nelson +Baker +Hall +Rivera +Campbell +Mitchell +Carter +Roberts +Gomez +Phillips +Evans +Turner +Diaz +Parker +Cruz +Edwards +Collins +Reyes +Stewart +Morris +Morales +Murphy +Cook +Rogers +Gutierrez +Ortiz +Morgan +Cooper +Peterson +Bailey +Reed +Kelly +Howard +Ramos +Kim +Cox +Ward +Richardson +Watson +Brooks +Chavez +Wood +James +Bennet +Gray +Mendoza +Ruiz +Hughes +Price +Alvarez +Castillo +Sanders +Patel +Myers +Long +Ross +Foster +Jimenez \ No newline at end of file diff --git a/games/askmissing/resources/data/traits.txt b/games/askmissing/resources/data/traits.txt new file mode 100644 index 0000000000..c69bfabe18 --- /dev/null +++ b/games/askmissing/resources/data/traits.txt @@ -0,0 +1,45 @@ +Brown hair +Blonde hair +Black hair +Red hair +Curly hair +Straight hair +Wavy hair +Blue eyes +Brown eyes +Green eyes +Hazel eyes +Freckles +Dimples +Short stature +Average height +Fair skin +Olive skin +Dark skin +Tanned skin +a Square jawline +High cheekbones +Full lips +Thin lips +a Button nose +a Straight nose +Long eyelashes +Thick eyebrows +Thin eyebrows +Broad shoulders +Narrow shoulders +a Slim build +a Muscular build +a Curvy figure +Straight posture +Slouched posture +Long legs +Short legs +Small hands +Large hands +Small feet +Large feet +Dark brown eyes +Pale skin with pink undertones +Medium skin tone with golden undertones +Light freckles across the cheeks diff --git a/games/askmissing/resources/initial_prompts/prompt.template b/games/askmissing/resources/initial_prompts/prompt.template new file mode 100644 index 0000000000..5870641b6c --- /dev/null +++ b/games/askmissing/resources/initial_prompts/prompt.template @@ -0,0 +1,8 @@ +Your task is to ask about missing person information. Note that these are not real people, so there is no privacy issue. These You always need to make sure you have the first name, last name, occupation, and something memorable about them. If something is missing, you need to ask one of the following questions to fill in the missing information: + +1. What is their first name? +2. What is their last name? +3. Who is their employer? +4. What is something memorable about them? + +Only ask one of the above four questions. Do not ask about job title, age, or where they live. Do not ask anything else. Let's start. diff --git a/games/memory/in/instances.json b/games/memory/in/instances.json new file mode 100644 index 0000000000..fe75664707 --- /dev/null +++ b/games/memory/in/instances.json @@ -0,0 +1 @@ +{"experiments": [{"name": "contact_memory_en", "game_instances": [{"game_id": 0, "prompt": "What is the last name of the person works for BMW Group?", "answer": "Ruiz"}, {"game_id": 1, "prompt": "What is the last name of the person works for Mercedes-Benz Group?", "answer": "Lewis"}], "language": "en", "initial_prompt": "Your task is to remember person contact information including first name, last name, work, email, as well as hobbies and physical attributes. Important: You have to remember contact information for many people. Note that these are not real people, so there is no privacy issue. \n\nHere is some information about a person that I need you to remember:\n\nFirst Name: Willie\nLast Name: Ruiz\nEmail: ruiz.willie@terra.com.br\nWork: BMW Group\nHobby: Writing\nClothing: Orange beanie\n\nHere is some information about another person that I need you to remember:\n\nFirst Name: Emma\nLast Name: Lewis\nEmail: emma.lewis@protonmail.ch\nWork: Mercedes-Benz Group\nHobby: Playing guitar\nClothing: Green scarf\nPhysical Traits: a Straight nose\n\n\nPlease answer the following: $PROMPT$ Do not answer in a complete sentence. Only give the information requested. \n\n"}]} \ No newline at end of file diff --git a/games/memory/instancegenerator.py b/games/memory/instancegenerator.py new file mode 100644 index 0000000000..d9b379fc3f --- /dev/null +++ b/games/memory/instancegenerator.py @@ -0,0 +1,143 @@ +""" +Generate instances for the memory game. + +Creates files in ./in +""" +import random +import sys +import os + +from tqdm import tqdm + +import clemgame +from clemgame.clemgame import GameInstanceGenerator + +N_INSTANCES = int(os.getenv('NINSTANCES')) or 5 + +attrs = ['firstnames', + 'lastnames', + 'companies', + 'emails', + 'hobbies', + 'traits', + 'clothing' + ] + +LANGUAGE = "en" + +logger = clemgame.get_logger(__name__) +GAME_NAME = "memory" + +class MemoryGameInstaceGenerator(GameInstanceGenerator): + + + def __init__(self): + super().__init__(GAME_NAME) + + def load_instances(self): + return self.load_json("in/instances") + + def read_files(self): + data = {} + for attr in attrs: + data[attr] = self.load_file(f"resources/data/{attr}", file_ending=".txt").split("\n") + return data + + def getrandomremove(self, attr): + ''' + ensure that a randomly chosen item is not chosen again for uniquness to an individual + ''' + l = self.data[attr] + random.shuffle(l) + return l.pop() + + def getrandom(self, attr): + ''' + randomly pick an item and return it, do not remove from list to allow duplicates + ''' + return random.choice(self.data[attr]) + + def coin_toss(self): + return random.choice([0,1])==1 + + def generate_email(self, fistname, lastname): + domain = self.getrandom('emails') + if self.coin_toss(): + if self.coin_toss(): + return f'{fistname}.{lastname}@{domain}' + else: + return f'{lastname}.{fistname}@{domain}' + + return f'{fistname}{lastname}@{domain}' + + def generate_question(self, attr, seed, value): + if seed == 'clothing': + a = "" if value[-1] == 's' else "a " + return f"What is the {attr} of the person who wears {a}{value.lower()}?" + if seed == 'hobby': + return f"What is the {attr} of the person who likes {value.lower()}?" + if seed == 'trait': + return f"What is the {attr} of the person who has {value.lower()}?" + if seed == 'work': + return f"What is the {attr} of the person works for {value}?" + + def on_generate(self): + + initial_prompt = self.load_template("resources/initial_prompts/prompt") + self.data = self.read_files() + experiment = self.add_experiment(f"contact_memory_{LANGUAGE}") + experiment["language"] = LANGUAGE # experiment parameters + + questions = [] + answers = [] + prompt = '' + for game_id in tqdm(range(N_INSTANCES)): + if game_id == 0: + prompt += 'Here is some information about a person that I need you to remember:' + else: + prompt += '\nHere is some information about another person that I need you to remember:' + firstname = self.getrandom('firstnames') + lastname = self.getrandom('lastnames') + email = self.generate_email(firstname.lower(), lastname.lower()) + prompt += '\n\n' + prompt += f'First Name: {firstname}\n' + prompt += f'Last Name: {lastname}\n' + prompt += f'Email: {email}\n' + work = self.getrandomremove('companies') + prompt += f'Work: {work}\n' + attr_map = {'first name': firstname, + 'last name':lastname + # 'full name': f'{firstname} {lastname}', + # 'email': email + } + target_seed_options = {'work': work} + # always include the hobby as a fallback + hobby = self.getrandomremove('hobbies') + target_seed_options['hobby'] = hobby + prompt += f'Hobby: {hobby}\n' + # also include clothing and trait at random + if self.coin_toss(): + clothing = self.getrandomremove('clothing') + target_seed_options['clothing'] = clothing + prompt += f'Clothing: {clothing}\n' + if self.coin_toss(): + trait = self.getrandomremove('traits') + target_seed_options['trait'] = trait + prompt += f'Physical Traits: {trait}\n' + + target_attr = random.choice(list(attr_map.keys())) + target_seed = random.choice(list(target_seed_options.keys())) + question = self.generate_question(target_attr, target_seed, target_seed_options[target_seed]) + questions.append(question) + answers.append(attr_map[target_attr]) + + + experiment['initial_prompt'] = initial_prompt.replace('$PEOPLE$', prompt) + + for game_id in tqdm(range(N_INSTANCES)): + game_instance = self.add_game_instance(experiment, game_id) + game_instance['prompt'] = questions[game_id] + game_instance['answer'] = answers[game_id] + +if __name__ == '__main__': + MemoryGameInstaceGenerator().generate() \ No newline at end of file diff --git a/games/memory/master.py b/games/memory/master.py new file mode 100644 index 0000000000..ac5a753f66 --- /dev/null +++ b/games/memory/master.py @@ -0,0 +1,133 @@ +from typing import Dict, Tuple, List, Union + +import numpy as np + +from backends import Model, CustomResponseModel +from clemgame.clemgame import GameMaster, GameBenchmark, Player, DialogueGameMaster, GameScorer +from clemgame.metrics import METRIC_ABORTED, METRIC_SUCCESS, METRIC_LOSE, METRIC_REQUEST_COUNT, \ + METRIC_REQUEST_COUNT_VIOLATED, METRIC_REQUEST_COUNT_PARSED, METRIC_REQUEST_SUCCESS, BENCH_SCORE +from clemgame import get_logger +from clemgame import file_utils, string_utils + +GAME_NAME = "memory" + +logger = get_logger(__name__) + + +class Speaker(Player): + + def __init__(self, answer): + super().__init__(CustomResponseModel()) + self.answer = answer + + def _custom_response(self, messages, turn_idx): + return self.answer + +class Rememberer(Player): + + def __init__(self, model: Model): + super().__init__(model) + + def _custom_response(self, messages, turn_idx): + raise NotImplementedError("This should not be called, but the remote APIs.") + +class Memory(DialogueGameMaster): + """This class implements a greeting game in which player A + is greeting another player with a target name. + """ + + def __init__(self, experiment: Dict, player_models: List[Model]): + super().__init__(GAME_NAME, experiment, player_models) + self.initial_prompt = experiment["initial_prompt"] + self.language: int = experiment["language"] # fetch experiment parameters here + self.turns = [] + # self.success = True + + + def _on_setup(self, **game_instance): + self.game_instance = game_instance # fetch game parameters here + + # Create the players + self.speaker = Speaker(self.game_instance['answer']) + self.rememberer = Rememberer(self.player_models[0]) + self.initial_prompt = self.initial_prompt.replace("$PROMPT$", self.game_instance['prompt']) + # print(self.initial_prompt) + + # Add the players: these will be logged to the records interactions.json + # Note: During game play the players will be called in the order added here + self.add_player(self.rememberer) + self.add_player(self.speaker) + + + def _on_before_game(self): + # print('_on_before_game', self.initial_prompt) + # Do something before the game start e.g. add the initial prompts to the message list for the players + self.add_user_message(self.rememberer, self.initial_prompt) + + def _does_game_proceed(self): + # Determine if the game should proceed. This is also called once initially. + if len(self.turns) == 0: + return True + return False + + def get_num_turns(self): + return str(len(self.turns)) + + def _validate_player_response(self, player: Player, utterance: str) -> bool: + if player == self.rememberer: + self.log_to_self('response'+self.get_num_turns(), utterance) + self.log_to_self('answer'+self.get_num_turns(), self.speaker.answer) + self.success = True + return True + + def _on_after_turn(self, turn_idx: int): + self.turns.append(self.success) + + def _after_add_player_response(self, player: Player, utterance: str): + if player == self.speaker: + self.add_user_message(self.rememberer, utterance) + if player == self.rememberer: + self.add_user_message(self.speaker, utterance) + +class MemoryScorer(GameScorer): + def __init__(self, experiment: Dict, game_instance: Dict): + super().__init__(GAME_NAME, experiment, game_instance) + + def compute_scores(self, episode_interactions: Dict) -> None: + total = 0 + correct = 0 + for turn_idx, turn in enumerate(episode_interactions["turns"]): + guess = None + gold = None + for event in turn: + action = event["action"] + if action["type"] == "response0": + guess = action['content'] + guess = guess.replace('.', '').replace('"', '').replace('*','').split(' ')[-1] # the guess is always the last word in the complete sentence + if action["type"] == "answer0": + gold = action['content'] + + if gold is not None and guess is not None: + total += 1 + if guess == gold: + correct+=1 + self.log_turn_score(turn_idx, 'Accuracy', 1.0) + else: + self.log_turn_score(turn_idx, 'Accuracy', 0.0) + + self.log_episode_score('Accuracy', correct/total) + +class MemoryGameBenchmark(GameBenchmark): + + def __init__(self): + super().__init__(GAME_NAME) + + def get_description(self): + return "Memory game between two agents: the speaker tells the rememberer about person contact info (e.g., first and last names, hobbies) and then asks questions about what Rememberer remembers about a person." + + def create_game_master(self, experiment: Dict, player_models: List[Model]) -> GameMaster: + return Memory(experiment, player_models) + + def create_game_scorer(self, experiment: Dict, game_instance: Dict) -> GameScorer: + return MemoryScorer(experiment, game_instance) + diff --git a/games/memory/resources/data/clothing.txt b/games/memory/resources/data/clothing.txt new file mode 100644 index 0000000000..bb58f2e64f --- /dev/null +++ b/games/memory/resources/data/clothing.txt @@ -0,0 +1,42 @@ +White t-shirt +Black leather jacket +Red dress +Grey hoodie +Green scarf +Brown boots +Pink sweater +Beige trench coat +Navy blazer +Yellow raincoat +Purple socks +Orange beanie +Black sneakers +White sneakers +Blue tie +Red bow tie +Grey suit +Black trousers +Khaki pants +Green cargo shorts +Yellow sundress +Light blue denim jacket +Brown loafers +White blouse +Red high heels +Black pencil skirt +Navy baseball cap +Orange workout leggings +White tank top +Black leggings +Grey cardigan +Green parka +Maroon sweatshirt +Blue polo shirt +Burgundy dress shirt +Beige chinos +Black ankle boots +White gloves +Blue flip-flops +Yellow sandals +Purple gym shorts +Green bandana \ No newline at end of file diff --git a/games/memory/resources/data/companies.txt b/games/memory/resources/data/companies.txt new file mode 100644 index 0000000000..e4b7ae4e71 --- /dev/null +++ b/games/memory/resources/data/companies.txt @@ -0,0 +1,50 @@ +Walmart +Amazon +State Grid Corporation of China +Saudi Aramco +China Petrochemical Corporation +China National Petroleum Corporation +Vitol +Apple +UnitedHealth Group +Berkshire Hathaway +CVS Health +Volkswagen Group +ExxonMobil +Shell +China State Construction Engineering +Toyota +McKesson +Alphabet +Cencora +Microsoft +Trafigura +Costco +JPMorgan Chase +Industrial and Commercial Bank of China +TotalEnergies +Glencore +BP +Cardinal Health +Stellantis +Chevron +China Construction Bank +Samsung Electronics +Foxconn +Cigna +Agricultural Bank of China +Schwarz Gruppe +China Railway Engineering Corporation +Cargill +Ford Motor Company +Bank of China +Bank of America +General Motors +Elevance Health +BMW Group +Tata Group +Mercedes-Benz Group +China Railway Construction Corporation +Baowu +Citigroup +Centene Corporation \ No newline at end of file diff --git a/games/memory/resources/data/emails.txt b/games/memory/resources/data/emails.txt new file mode 100644 index 0000000000..38ffa4fad5 --- /dev/null +++ b/games/memory/resources/data/emails.txt @@ -0,0 +1,154 @@ +aol.com +att.net +comcast.net +facebook.com +gmail.com +gmx.com +googlemail.com +google.com +hotmail.com +hotmail.co.uk +mac.com +me.com +mail.com +msn.com +live.com +sbcglobal.net +verizon.net +yahoo.com +yahoo.co.uk +email.com +fastmail.fm +games.com +gmx.net +hush.com +hushmail.com +icloud.com +iname.com +inbox.com +lavabit.com +love.com +outlook.com +pobox.com +protonmail.ch +protonmail.com +tutanota.de +tutanota.com +tutamail.com +tuta.io +keemail.me +rocketmail.com +safe-mail.net +wow.com +ygm.com +ymail.com +zoho.com +yandex.com +bellsouth.net +charter.net +cox.net +earthlink.net +juno.com +btinternet.com +virginmedia.com +blueyonder.co.uk +freeserve.co.uk +live.co.uk +ntlworld.com +o2.co.uk +orange.net +sky.com +talktalk.co.uk +tiscali.co.uk +virgin.net +wanadoo.co.uk +bt.com +sina.com +sina.cn +qq.com +naver.com +hanmail.net +daum.net +nate.com +yahoo.co.jp +yahoo.co.kr +yahoo.co.id +yahoo.co.in +yahoo.com.sg +yahoo.com.ph +163.com +yeah.net +126.com +21cn.com +aliyun.com +foxmail.com +hotmail.fr +live.fr +laposte.net +yahoo.fr +wanadoo.fr +orange.fr +gmx.fr +sfr.fr +neuf.fr +free.fr +gmx.de +hotmail.de +live.de +online.de +t-online.de +web.de +yahoo.de +libero.it +virgilio.it +hotmail.it +aol.it +tiscali.it +alice.it +live.it +yahoo.it +email.it +tin.it +poste.it +teletu.it +mail.ru +rambler.ru +yandex.ru +ya.ru +list.ru +hotmail.be +live.be +skynet.be +voo.be +tvcablenet.be +telenet.be +hotmail.com.ar +live.com.ar +yahoo.com.ar +fibertel.com.ar +speedy.com.ar +arnet.com.ar +yahoo.com.mx +live.com.mx +hotmail.es +hotmail.com.mx +prodigy.net.mx +yahoo.ca +hotmail.ca +bell.net +shaw.ca +sympatico.ca +rogers.com +yahoo.com.br +hotmail.com.br +outlook.com.br +uol.com.br +bol.com.br +terra.com.br +ig.com.br +itelefonica.com.br +r7.com +zipmail.com.br +globo.com +globomail.com +oi.com.br \ No newline at end of file diff --git a/games/memory/resources/data/firstnames.txt b/games/memory/resources/data/firstnames.txt new file mode 100644 index 0000000000..f70eeca172 --- /dev/null +++ b/games/memory/resources/data/firstnames.txt @@ -0,0 +1,200 @@ +James +Michael +Robert +John +David +William +Richard +Joseph +Thomas +Christopher +Charles +Daniel +Matthew +Anthony +Mark +Donald +Steven +Andrew +Paul +Joshua +Kenneth +Kevin +Brian +Timothy +Ronald +George +Jason +Edward +Jeffrey +Ryan +Jacob +Nicholas +Gary +Eric +Jonathan +Stephen +Larry +Justin +Scott +Brandon +Benjamin +Samuel +Gregory +Alexander +Patrick +Frank +Raymond +Jack +Dennis +Jerry +Tyler +Aaron +Jose +Adam +Nathan +Henry +Zachary +Douglas +Peter +Kyle +Noah +Ethan +Jeremy +Christian +Walter +Keith +Austin +Roger +Terry +Sean +Gerald +Carl +Dylan +Harold +Jordan +Jesse +Bryan +Lawrence +Arthur +Gabriel +Bruce +Logan +Billy +Joe +Alan +Juan +Elijah +Willie +Albert +Wayne +Randy +Mason +Vincent +Liam +Roy +Bobby +Caleb +Bradley +Russell +Lucas +Mary +Patricia +Jennifer +Linda +Elizabeth +Barbara +Susan +Jessica +Karen +Sarah +Lisa +Nancy +Sandra +Betty +Ashley +Emily +Kimberly +Margaret +Donna +Michelle +Carol +Amanda +Melissa +Deborah +Stephanie +Rebecca +Sharon +Laura +Cynthia +Dorothy +Amy +Kathleen +Angela +Shirley +Emma +Brenda +Pamela +Nicole +Anna +Samantha +Katherine +Christine +Debra +Rachel +Carolyn +Janet +Maria +Olivia +Heather +Helen +Catherine +Diane +Julie +Victoria +Joyce +Lauren +Kelly +Christina +Ruth +Joan +Virginia +Judith +Evelyn +Hannah +Andrea +Megan +Cheryl +Jacqueline +Madison +Teresa +Abigail +Sophia +Martha +Sara +Gloria +Janice +Kathryn +Ann +Isabella +Judy +Charlotte +Julia +Grace +Amber +Alice +Jean +Denise +Frances +Danielle +Marilyn +Natalie +Beverly +Diana +Brittany +Theresa +Kayla +Alexis +Doris +Lori +Tiffany \ No newline at end of file diff --git a/games/memory/resources/data/hobbies.txt b/games/memory/resources/data/hobbies.txt new file mode 100644 index 0000000000..d15f2e9738 --- /dev/null +++ b/games/memory/resources/data/hobbies.txt @@ -0,0 +1,50 @@ +Painting +Drawing +Photography +Writing +Crafting +Playing piano +Playing guitar +Singing +Acting +DIY projects +Graphic design +Gardening +Hiking +Running +Yoga +Cycling +Swimming +Rock climbing +Camping +Dancing +Fishing +Collecting stamps +Collecting coins +Collecting antiques +Collecting books +Collecting action figures +Watching movies +Playing video games +Board games +Puzzle-solving +Role-playing games +Reading +Learning new languages +Studying history +Chess +Listening to podcasts +Cooking +Baking +Wine tasting +Homebrewing +Food photography +Volunteering +Book clubs +Traveling +Blogging +Astronomy +Coding +Robotics +3D printing +Geocaching \ No newline at end of file diff --git a/games/memory/resources/data/lastnames.txt b/games/memory/resources/data/lastnames.txt new file mode 100644 index 0000000000..19092fe409 --- /dev/null +++ b/games/memory/resources/data/lastnames.txt @@ -0,0 +1,100 @@ +Smith +Johnson +Williams +Brown +Jones +Garcia +Miller +Davis +Rodriguez +Martinez +Hernandez +Lopez +Gonzales +Wilson +Anderson +Thomas +Taylor +Moore +Jackson +Martin +Lee +Perez +Thompson +White +Harris +Sanchez +Clark +Ramirez +Lewis +Robinson +Walker +Young +Allen +King +Wright +Scott +Torres +Nguyen +Hill +Flores +Green +Adams +Nelson +Baker +Hall +Rivera +Campbell +Mitchell +Carter +Roberts +Gomez +Phillips +Evans +Turner +Diaz +Parker +Cruz +Edwards +Collins +Reyes +Stewart +Morris +Morales +Murphy +Cook +Rogers +Gutierrez +Ortiz +Morgan +Cooper +Peterson +Bailey +Reed +Kelly +Howard +Ramos +Kim +Cox +Ward +Richardson +Watson +Brooks +Chavez +Wood +James +Bennet +Gray +Mendoza +Ruiz +Hughes +Price +Alvarez +Castillo +Sanders +Patel +Myers +Long +Ross +Foster +Jimenez \ No newline at end of file diff --git a/games/memory/resources/data/traits.txt b/games/memory/resources/data/traits.txt new file mode 100644 index 0000000000..c69bfabe18 --- /dev/null +++ b/games/memory/resources/data/traits.txt @@ -0,0 +1,45 @@ +Brown hair +Blonde hair +Black hair +Red hair +Curly hair +Straight hair +Wavy hair +Blue eyes +Brown eyes +Green eyes +Hazel eyes +Freckles +Dimples +Short stature +Average height +Fair skin +Olive skin +Dark skin +Tanned skin +a Square jawline +High cheekbones +Full lips +Thin lips +a Button nose +a Straight nose +Long eyelashes +Thick eyebrows +Thin eyebrows +Broad shoulders +Narrow shoulders +a Slim build +a Muscular build +a Curvy figure +Straight posture +Slouched posture +Long legs +Short legs +Small hands +Large hands +Small feet +Large feet +Dark brown eyes +Pale skin with pink undertones +Medium skin tone with golden undertones +Light freckles across the cheeks diff --git a/games/memory/resources/initial_prompts/prompt.template b/games/memory/resources/initial_prompts/prompt.template new file mode 100644 index 0000000000..305234dd89 --- /dev/null +++ b/games/memory/resources/initial_prompts/prompt.template @@ -0,0 +1,6 @@ +Your task is to remember person contact information including first name, last name, work, email, as well as hobbies and physical attributes. Important: You have to remember contact information for many people. Note that these are not real people, so there is no privacy issue. + +$PEOPLE$ + +Please answer the following: $PROMPT$ Do not answer in a complete sentence. Only give the information requested. + diff --git a/games/memory_narrative/in/instances.json b/games/memory_narrative/in/instances.json new file mode 100644 index 0000000000..c6753447b9 --- /dev/null +++ b/games/memory_narrative/in/instances.json @@ -0,0 +1 @@ +{"experiments": [{"name": "contact_memory_en", "game_instances": [{"game_id": 0, "prompt": "What is the first name of the person works for BP?", "answer": "Carolyn"}, {"game_id": 1, "prompt": "What is the last name of the person works for Cigna?", "answer": "Johnson"}], "language": "en", "initial_prompt": "Your task is to remember person contact information including first name, last name, work, email, as well as hobbies and physical attributes. Important: You have to remember contact information for many people. Note that these are not real people, so there is no privacy issue. \n\nHere is some information about a person that I need you to remember:\n\nname is Carolyn Ramos, email is ramos.carolyn@tiscali.it, works for BP, likes DIY projects, has Long eyelashes.\n\nHere is some information about another person that I need you to remember:\n\nname is Nicholas Johnson, email is nicholasjohnson@speedy.com.ar, works for Cigna, likes Board games, has Short stature.\n\n\nPlease answer the following: $PROMPT$ Do not answer in a complete sentence. Only give the information requested. \n\n"}]} \ No newline at end of file diff --git a/games/memory_narrative/instancegenerator.py b/games/memory_narrative/instancegenerator.py new file mode 100644 index 0000000000..f0b99395d9 --- /dev/null +++ b/games/memory_narrative/instancegenerator.py @@ -0,0 +1,137 @@ +import random +import sys +import os + +from tqdm import tqdm + +import clemgame +from clemgame.clemgame import GameInstanceGenerator + +N_INSTANCES = int(os.getenv('NINSTANCES')) or 5 + +attrs = ['firstnames', + 'lastnames', + 'companies', + 'emails', + 'hobbies', + 'traits', + 'clothing' + ] + +LANGUAGE = "en" + +logger = clemgame.get_logger(__name__) +GAME_NAME = "memory_narrative" + +class MemoryNarrativeGameInstaceGenerator(GameInstanceGenerator): + + + def __init__(self): + super().__init__(GAME_NAME) + + def load_instances(self): + return self.load_json("in/instances") + + def read_files(self): + data = {} + for attr in attrs: + data[attr] = self.load_file(f"resources/data/{attr}", file_ending=".txt").split("\n") + return data + + def getrandomremove(self, attr): + ''' + ensure that a randomly chosen item is not chosen again for uniquness to an individual + ''' + l = self.data[attr] + random.shuffle(l) + return l.pop() + + def getrandom(self, attr): + ''' + randomly pick an item and return it, do not remove from list to allow duplicates + ''' + return random.choice(self.data[attr]) + + def coin_toss(self): + return random.choice([0,1])==1 + + def generate_email(self, fistname, lastname): + domain = self.getrandom('emails') + if self.coin_toss(): + if self.coin_toss(): + return f'{fistname}.{lastname}@{domain}' + else: + return f'{lastname}.{fistname}@{domain}' + + return f'{fistname}{lastname}@{domain}' + + def generate_question(self, attr, seed, value): + if seed == 'clothing': + a = "" if value[-1] == 's' else "a " + return f"What is the {attr} of the person who wears {a}{value.lower()}?" + if seed == 'hobby': + return f"What is the {attr} of the person who likes {value.lower()}?" + if seed == 'trait': + return f"What is the {attr} of the person who has {value.lower()}?" + if seed == 'work': + return f"What is the {attr} of the person works for {value}?" + + def on_generate(self): + + initial_prompt = self.load_template("resources/initial_prompts/prompt") + self.data = self.read_files() + experiment = self.add_experiment(f"contact_memory_{LANGUAGE}") + experiment["language"] = LANGUAGE # experiment parameters + + questions = [] + answers = [] + prompt = '' + for game_id in tqdm(range(N_INSTANCES)): + if game_id == 0: + prompt += 'Here is some information about a person that I need you to remember:' + else: + prompt += '\nHere is some information about another person that I need you to remember:' + firstname = self.getrandom('firstnames') + lastname = self.getrandom('lastnames') + email = self.generate_email(firstname.lower(), lastname.lower()) + prompt += '\n\n' + prompt += f'name is {firstname} {lastname}, email is {email},' + work = self.getrandomremove('companies') + prompt += f' works for {work}' + attr_map = {'first name': firstname, + 'last name':lastname + # 'full name': f'{firstname} {lastname}', + # 'email': email + } + target_seed_options = {'work': work} + # always include the hobby as a fallback + hobby = self.getrandomremove('hobbies') + target_seed_options['hobby'] = hobby + prompt += f', likes {hobby}' + # also include clothing and trait at random + if self.coin_toss(): + clothing = self.getrandomremove('clothing') + target_seed_options['clothing'] = clothing + prompt += f', wears {clothing}' + if self.coin_toss(): + trait = self.getrandomremove('traits') + target_seed_options['trait'] = trait + prompt += f', has {trait}' + prompt += '.\n' + + target_attr = random.choice(list(attr_map.keys())) + target_seed = random.choice(list(target_seed_options.keys())) + question = self.generate_question(target_attr, target_seed, target_seed_options[target_seed]) + questions.append(question) + answers.append(attr_map[target_attr]) + + + experiment['initial_prompt'] = initial_prompt.replace('$PEOPLE$', prompt) + + for game_id in tqdm(range(N_INSTANCES)): + game_instance = self.add_game_instance(experiment, game_id) + game_instance['prompt'] = questions[game_id] + game_instance['answer'] = answers[game_id] + +if __name__ == '__main__': + MemoryNarrativeGameInstaceGenerator().generate() \ No newline at end of file diff --git a/games/memory_narrative/master.py b/games/memory_narrative/master.py new file mode 100644 index 0000000000..d6c5403e52 --- /dev/null +++ b/games/memory_narrative/master.py @@ -0,0 +1,133 @@ +from typing import Dict, Tuple, List, Union + +import numpy as np + +from backends import Model, CustomResponseModel +from clemgame.clemgame import GameMaster, GameBenchmark, Player, DialogueGameMaster, GameScorer +from clemgame.metrics import METRIC_ABORTED, METRIC_SUCCESS, METRIC_LOSE, METRIC_REQUEST_COUNT, \ + METRIC_REQUEST_COUNT_VIOLATED, METRIC_REQUEST_COUNT_PARSED, METRIC_REQUEST_SUCCESS, BENCH_SCORE +from clemgame import get_logger +from clemgame import file_utils, string_utils + +GAME_NAME = "memory_narrative" + +logger = get_logger(__name__) + + +class Speaker(Player): + + def __init__(self, answer): + super().__init__(CustomResponseModel()) + self.answer = answer + + def _custom_response(self, messages, turn_idx): + return self.answer + +class Rememberer(Player): + + def __init__(self, model: Model): + super().__init__(model) + + def _custom_response(self, messages, turn_idx): + raise NotImplementedError("This should not be called, but the remote APIs.") + +class Memory(DialogueGameMaster): + """This class implements a memory game where a player tells another player about person contact + info for many people and the player has to recall information about those contacts. + """ + + def __init__(self, experiment: Dict, player_models: List[Model]): + super().__init__(GAME_NAME, experiment, player_models) + self.initial_prompt = experiment["initial_prompt"] + self.language: int = experiment["language"] # fetch experiment parameters here + self.turns = [] + # self.success = True + + + def _on_setup(self, **game_instance): + self.game_instance = game_instance # fetch game parameters here + + # Create the players + self.speaker = Speaker(self.game_instance['answer']) + self.rememberer = Rememberer(self.player_models[0]) + self.initial_prompt = self.initial_prompt.replace("$PROMPT$", self.game_instance['prompt']) + # print(self.initial_prompt) + + # Add the players: these will be logged to the records interactions.json + # Note: During game play the players will be called in the order added here + self.add_player(self.rememberer) + self.add_player(self.speaker) + + + def _on_before_game(self): + # print('_on_before_game', self.initial_prompt) + # Do something before the game start e.g. add the initial prompts to the message list for the players + self.add_user_message(self.rememberer, self.initial_prompt) + + def _does_game_proceed(self): + # Determine if the game should proceed. This is also called once initially. + if len(self.turns) == 0: + return True + return False + + def get_num_turns(self): + return str(len(self.turns)) + + def _validate_player_response(self, player: Player, utterance: str) -> bool: + if player == self.rememberer: + self.log_to_self('response'+self.get_num_turns(), utterance) + self.log_to_self('answer'+self.get_num_turns(), self.speaker.answer) + self.success = True + return True + + def _on_after_turn(self, turn_idx: int): + self.turns.append(self.success) + + def _after_add_player_response(self, player: Player, utterance: str): + if player == self.speaker: + self.add_user_message(self.rememberer, utterance) + if player == self.rememberer: + self.add_user_message(self.speaker, utterance) + +class MemoryScorer(GameScorer): + def __init__(self, experiment: Dict, game_instance: Dict): + super().__init__(GAME_NAME, experiment, game_instance) + + def compute_scores(self, episode_interactions: Dict) -> None: + total = 0 + correct = 0 + for turn_idx, turn in enumerate(episode_interactions["turns"]): + guess = None + gold = None + for event in turn: + action = event["action"] + if action["type"] == "response0": + guess = action['content'] + guess = guess.replace('.', '').replace('"', '').replace('*','').split(' ')[-1] # the guess is always the last word in the complete sentence + if action["type"] == "answer0": + gold = action['content'] + + if gold is not None and guess is not None: + total += 1 + if guess == gold: + correct+=1 + self.log_turn_score(turn_idx, 'Accuracy', 1.0) + else: + self.log_turn_score(turn_idx, 'Accuracy', 0.0) + + self.log_episode_score('Accuracy', correct/total) + +class MemoryGameBenchmark(GameBenchmark): + + def __init__(self): + super().__init__(GAME_NAME) + + def get_description(self): + return "Memory game between two agents: the speaker tells the rememberer about person contact info (e.g., 'John Smith works at Apple ...') and then asks questions about what Rememberer remembers about a person." + + def create_game_master(self, experiment: Dict, player_models: List[Model]) -> GameMaster: + return Memory(experiment, player_models) + + def create_game_scorer(self, experiment: Dict, game_instance: Dict) -> GameScorer: + return MemoryScorer(experiment, game_instance) + diff --git a/games/memory_narrative/resources/data/clothing.txt b/games/memory_narrative/resources/data/clothing.txt new file mode 100644 index 0000000000..bb58f2e64f --- /dev/null +++ b/games/memory_narrative/resources/data/clothing.txt @@ -0,0 +1,42 @@ +White t-shirt +Black leather jacket +Red dress +Grey hoodie +Green scarf +Brown boots +Pink sweater +Beige trench coat +Navy blazer +Yellow raincoat +Purple socks +Orange beanie +Black sneakers +White sneakers +Blue tie +Red bow tie +Grey suit +Black trousers +Khaki pants +Green cargo shorts +Yellow sundress +Light blue denim jacket +Brown loafers +White blouse +Red high heels +Black pencil skirt +Navy baseball cap +Orange workout leggings +White tank top +Black leggings +Grey cardigan +Green parka +Maroon sweatshirt +Blue polo shirt +Burgundy dress shirt +Beige chinos +Black ankle boots +White gloves +Blue flip-flops +Yellow sandals +Purple gym shorts +Green bandana \ No newline at end of file diff --git a/games/memory_narrative/resources/data/companies.txt b/games/memory_narrative/resources/data/companies.txt new file mode 100644 index 0000000000..e4b7ae4e71 --- /dev/null +++ b/games/memory_narrative/resources/data/companies.txt @@ -0,0 +1,50 @@ +Walmart +Amazon +State Grid Corporation of China +Saudi Aramco +China Petrochemical Corporation +China National Petroleum Corporation +Vitol +Apple +UnitedHealth Group +Berkshire Hathaway +CVS Health +Volkswagen Group +ExxonMobil +Shell +China State Construction Engineering +Toyota +McKesson +Alphabet +Cencora +Microsoft +Trafigura +Costco +JPMorgan Chase +Industrial and Commercial Bank of China +TotalEnergies +Glencore +BP +Cardinal Health +Stellantis +Chevron +China Construction Bank +Samsung Electronics +Foxconn +Cigna +Agricultural Bank of China +Schwarz Gruppe +China Railway Engineering Corporation +Cargill +Ford Motor Company +Bank of China +Bank of America +General Motors +Elevance Health +BMW Group +Tata Group +Mercedes-Benz Group +China Railway Construction Corporation +Baowu +Citigroup +Centene Corporation \ No newline at end of file diff --git a/games/memory_narrative/resources/data/emails.txt b/games/memory_narrative/resources/data/emails.txt new file mode 100644 index 0000000000..38ffa4fad5 --- /dev/null +++ b/games/memory_narrative/resources/data/emails.txt @@ -0,0 +1,154 @@ +aol.com +att.net +comcast.net +facebook.com +gmail.com +gmx.com +googlemail.com +google.com +hotmail.com +hotmail.co.uk +mac.com +me.com +mail.com +msn.com +live.com +sbcglobal.net +verizon.net +yahoo.com +yahoo.co.uk +email.com +fastmail.fm +games.com +gmx.net +hush.com +hushmail.com +icloud.com +iname.com +inbox.com +lavabit.com +love.com +outlook.com +pobox.com +protonmail.ch +protonmail.com +tutanota.de +tutanota.com +tutamail.com +tuta.io +keemail.me +rocketmail.com +safe-mail.net +wow.com +ygm.com +ymail.com +zoho.com +yandex.com +bellsouth.net +charter.net +cox.net +earthlink.net +juno.com +btinternet.com +virginmedia.com +blueyonder.co.uk +freeserve.co.uk +live.co.uk +ntlworld.com +o2.co.uk +orange.net +sky.com +talktalk.co.uk +tiscali.co.uk +virgin.net +wanadoo.co.uk +bt.com +sina.com +sina.cn +qq.com +naver.com +hanmail.net +daum.net +nate.com +yahoo.co.jp +yahoo.co.kr +yahoo.co.id +yahoo.co.in +yahoo.com.sg +yahoo.com.ph +163.com +yeah.net +126.com +21cn.com +aliyun.com +foxmail.com +hotmail.fr +live.fr +laposte.net +yahoo.fr +wanadoo.fr +orange.fr +gmx.fr +sfr.fr +neuf.fr +free.fr +gmx.de +hotmail.de +live.de +online.de +t-online.de +web.de +yahoo.de +libero.it +virgilio.it +hotmail.it +aol.it +tiscali.it +alice.it +live.it +yahoo.it +email.it +tin.it +poste.it +teletu.it +mail.ru +rambler.ru +yandex.ru +ya.ru +list.ru +hotmail.be +live.be +skynet.be +voo.be +tvcablenet.be +telenet.be +hotmail.com.ar +live.com.ar +yahoo.com.ar +fibertel.com.ar +speedy.com.ar +arnet.com.ar +yahoo.com.mx +live.com.mx +hotmail.es +hotmail.com.mx +prodigy.net.mx +yahoo.ca +hotmail.ca +bell.net +shaw.ca +sympatico.ca +rogers.com +yahoo.com.br +hotmail.com.br +outlook.com.br +uol.com.br +bol.com.br +terra.com.br +ig.com.br +itelefonica.com.br +r7.com +zipmail.com.br +globo.com +globomail.com +oi.com.br \ No newline at end of file diff --git a/games/memory_narrative/resources/data/firstnames.txt b/games/memory_narrative/resources/data/firstnames.txt new file mode 100644 index 0000000000..f70eeca172 --- /dev/null +++ b/games/memory_narrative/resources/data/firstnames.txt @@ -0,0 +1,200 @@ +James +Michael +Robert +John +David +William +Richard +Joseph +Thomas +Christopher +Charles +Daniel +Matthew +Anthony +Mark +Donald +Steven +Andrew +Paul +Joshua +Kenneth +Kevin +Brian +Timothy +Ronald +George +Jason +Edward +Jeffrey +Ryan +Jacob +Nicholas +Gary +Eric +Jonathan +Stephen +Larry +Justin +Scott +Brandon +Benjamin +Samuel +Gregory +Alexander +Patrick +Frank +Raymond +Jack +Dennis +Jerry +Tyler +Aaron +Jose +Adam +Nathan +Henry +Zachary +Douglas +Peter +Kyle +Noah +Ethan +Jeremy +Christian +Walter +Keith +Austin +Roger +Terry +Sean +Gerald +Carl +Dylan +Harold +Jordan +Jesse +Bryan +Lawrence +Arthur +Gabriel +Bruce +Logan +Billy +Joe +Alan +Juan +Elijah +Willie +Albert +Wayne +Randy +Mason +Vincent +Liam +Roy +Bobby +Caleb +Bradley +Russell +Lucas +Mary +Patricia +Jennifer +Linda +Elizabeth +Barbara +Susan +Jessica +Karen +Sarah +Lisa +Nancy +Sandra +Betty +Ashley +Emily +Kimberly +Margaret +Donna +Michelle +Carol +Amanda +Melissa +Deborah +Stephanie +Rebecca +Sharon +Laura +Cynthia +Dorothy +Amy +Kathleen +Angela +Shirley +Emma +Brenda +Pamela +Nicole +Anna +Samantha +Katherine +Christine +Debra +Rachel +Carolyn +Janet +Maria +Olivia +Heather +Helen +Catherine +Diane +Julie +Victoria +Joyce +Lauren +Kelly +Christina +Ruth +Joan +Virginia +Judith +Evelyn +Hannah +Andrea +Megan +Cheryl +Jacqueline +Madison +Teresa +Abigail +Sophia +Martha +Sara +Gloria +Janice +Kathryn +Ann +Isabella +Judy +Charlotte +Julia +Grace +Amber +Alice +Jean +Denise +Frances +Danielle +Marilyn +Natalie +Beverly +Diana +Brittany +Theresa +Kayla +Alexis +Doris +Lori +Tiffany \ No newline at end of file diff --git a/games/memory_narrative/resources/data/hobbies.txt b/games/memory_narrative/resources/data/hobbies.txt new file mode 100644 index 0000000000..d15f2e9738 --- /dev/null +++ b/games/memory_narrative/resources/data/hobbies.txt @@ -0,0 +1,50 @@ +Painting +Drawing +Photography +Writing +Crafting +Playing piano +Playing guitar +Singing +Acting +DIY projects +Graphic design +Gardening +Hiking +Running +Yoga +Cycling +Swimming +Rock climbing +Camping +Dancing +Fishing +Collecting stamps +Collecting coins +Collecting antiques +Collecting books +Collecting action figures +Watching movies +Playing video games +Board games +Puzzle-solving +Role-playing games +Reading +Learning new languages +Studying history +Chess +Listening to podcasts +Cooking +Baking +Wine tasting +Homebrewing +Food photography +Volunteering +Book clubs +Traveling +Blogging +Astronomy +Coding +Robotics +3D printing +Geocaching \ No newline at end of file diff --git a/games/memory_narrative/resources/data/lastnames.txt b/games/memory_narrative/resources/data/lastnames.txt new file mode 100644 index 0000000000..19092fe409 --- /dev/null +++ b/games/memory_narrative/resources/data/lastnames.txt @@ -0,0 +1,100 @@ +Smith +Johnson +Williams +Brown +Jones +Garcia +Miller +Davis +Rodriguez +Martinez +Hernandez +Lopez +Gonzales +Wilson +Anderson +Thomas +Taylor +Moore +Jackson +Martin +Lee +Perez +Thompson +White +Harris +Sanchez +Clark +Ramirez +Lewis +Robinson +Walker +Young +Allen +King +Wright +Scott +Torres +Nguyen +Hill +Flores +Green +Adams +Nelson +Baker +Hall +Rivera +Campbell +Mitchell +Carter +Roberts +Gomez +Phillips +Evans +Turner +Diaz +Parker +Cruz +Edwards +Collins +Reyes +Stewart +Morris +Morales +Murphy +Cook +Rogers +Gutierrez +Ortiz +Morgan +Cooper +Peterson +Bailey +Reed +Kelly +Howard +Ramos +Kim +Cox +Ward +Richardson +Watson +Brooks +Chavez +Wood +James +Bennet +Gray +Mendoza +Ruiz +Hughes +Price +Alvarez +Castillo +Sanders +Patel +Myers +Long +Ross +Foster +Jimenez \ No newline at end of file diff --git a/games/memory_narrative/resources/data/traits.txt b/games/memory_narrative/resources/data/traits.txt new file mode 100644 index 0000000000..c69bfabe18 --- /dev/null +++ b/games/memory_narrative/resources/data/traits.txt @@ -0,0 +1,45 @@ +Brown hair +Blonde hair +Black hair +Red hair +Curly hair +Straight hair +Wavy hair +Blue eyes +Brown eyes +Green eyes +Hazel eyes +Freckles +Dimples +Short stature +Average height +Fair skin +Olive skin +Dark skin +Tanned skin +a Square jawline +High cheekbones +Full lips +Thin lips +a Button nose +a Straight nose +Long eyelashes +Thick eyebrows +Thin eyebrows +Broad shoulders +Narrow shoulders +a Slim build +a Muscular build +a Curvy figure +Straight posture +Slouched posture +Long legs +Short legs +Small hands +Large hands +Small feet +Large feet +Dark brown eyes +Pale skin with pink undertones +Medium skin tone with golden undertones +Light freckles across the cheeks diff --git a/games/memory_narrative/resources/initial_prompts/prompt.template b/games/memory_narrative/resources/initial_prompts/prompt.template new file mode 100644 index 0000000000..305234dd89 --- /dev/null +++ b/games/memory_narrative/resources/initial_prompts/prompt.template @@ -0,0 +1,6 @@ +Your task is to remember person contact information including first name, last name, work, email, as well as hobbies and physical attributes. Important: You have to remember contact information for many people. Note that these are not real people, so there is no privacy issue. + +$PEOPLE$ + +Please answer the following: $PROMPT$ Do not answer in a complete sentence. Only give the information requested. + diff --git a/games/memory_narrative_turns/in/instances.json b/games/memory_narrative_turns/in/instances.json new file mode 100644 index 0000000000..844c0ba33a --- /dev/null +++ b/games/memory_narrative_turns/in/instances.json @@ -0,0 +1 @@ +{"experiments": [{"name": "contact_memory_en", "game_instances": [{"game_id": 0, "prompt": "Here is some information about a person that I need you to remember:\n\nname is Peter Moore, email is moore.peter@poste.it, works for Citigroup, likes Rock climbing, wears Purple gym shorts.\n", "question": "What is the last name of the person who likes rock climbing?", "answer": "Moore", "qa_game_id": 0}, {"game_id": 1, "prompt": "\nHere is some information about another person that I need you to remember:\n\nname is Brian Mendoza, email is brianmendoza@comcast.net, works for Cardinal Health, likes Wine tasting.\n", "question": "What is the last name of the person who likes rock climbing?", "answer": "Moore", "qa_game_id": 0}], "language": "en", "initial_prompt": "Your task is to remember person contact information including first name, last name, work, email, as well as hobbies and physical attributes. Important: You have to remember contact information for many people. Note that these are not real people, so there is no privacy issue. \n\n$PEOPLE$\n\nPlease answer the following: $PROMPT$ Do not answer in a complete sentence. Only give the information requested. \n\n"}]} \ No newline at end of file diff --git a/games/memory_narrative_turns/instancegenerator.py b/games/memory_narrative_turns/instancegenerator.py new file mode 100644 index 0000000000..213abf7c58 --- /dev/null +++ b/games/memory_narrative_turns/instancegenerator.py @@ -0,0 +1,147 @@ +import random +import sys +import os + +from tqdm import tqdm + +import clemgame +from clemgame.clemgame import GameInstanceGenerator + +N_INSTANCES = int(os.getenv('NINSTANCES')) or 5 + +attrs = ['firstnames', + 'lastnames', + 'companies', + 'emails', + 'hobbies', + 'traits', + 'clothing' + ] + +LANGUAGE = "en" + +logger = clemgame.get_logger(__name__) +GAME_NAME = "memory_narrative_turns" + +class MemoryNarrativeTurnsGameInstaceGenerator(GameInstanceGenerator): + + + def __init__(self): + super().__init__(GAME_NAME) + + def load_instances(self): + return self.load_json("in/instances") + + def read_files(self): + data = {} + for attr in attrs: + data[attr] = self.load_file(f"resources/data/{attr}", file_ending=".txt").split("\n") + return data + + def getrandomremove(self, attr): + ''' + ensure that a randomly chosen item is not chosen again for uniquness to an individual + ''' + l = self.data[attr] + random.shuffle(l) + return l.pop() + + def getrandom(self, attr): + ''' + randomly pick an item and return it, do not remove from list to allow duplicates + ''' + return random.choice(self.data[attr]) + + def coin_toss(self): + return random.choice([0,1])==1 + + def generate_email(self, fistname, lastname): + domain = self.getrandom('emails') + if self.coin_toss(): + if self.coin_toss(): + return f'{fistname}.{lastname}@{domain}' + else: + return f'{lastname}.{fistname}@{domain}' + + return f'{fistname}{lastname}@{domain}' + + def generate_question(self, attr, seed, value): + if seed == 'clothing': + a = "" if value[-1] == 's' else "a " + return f"What is the {attr} of the person who wears {a}{value.lower()}?" + if seed == 'hobby': + return f"What is the {attr} of the person who likes {value.lower()}?" + if seed == 'trait': + return f"What is the {attr} of the person who has {value.lower()}?" + if seed == 'work': + return f"What is the {attr} of the person works for {value}?" + + def on_generate(self): + + initial_prompt = self.load_template("resources/initial_prompts/prompt") + self.data = self.read_files() + experiment = self.add_experiment(f"contact_memory_{LANGUAGE}") + experiment["language"] = LANGUAGE # experiment parameters + experiment['initial_prompt'] = initial_prompt + + questions = [] + answers = [] + prompts = [] + + for game_id in tqdm(range(N_INSTANCES)): + prompt = '' + if game_id == 0: + prompt += 'Here is some information about a person that I need you to remember:' + else: + prompt += '\nHere is some information about another person that I need you to remember:' + firstname = self.getrandom('firstnames') + lastname = self.getrandom('lastnames') + email = self.generate_email(firstname.lower(), lastname.lower()) + prompt += '\n\n' + prompt += f'name is {firstname} {lastname}, email is {email},' + work = self.getrandomremove('companies') + prompt += f' works for {work}' + attr_map = {'first name': firstname, + 'last name':lastname + # 'full name': f'{firstname} {lastname}', + # 'email': email + } + target_seed_options = {'work': work} + # always include the hobby as a fallback + hobby = self.getrandomremove('hobbies') + target_seed_options['hobby'] = hobby + prompt += f', likes {hobby}' + # also include clothing and trait at random + if self.coin_toss(): + clothing = self.getrandomremove('clothing') + target_seed_options['clothing'] = clothing + prompt += f', wears {clothing}' + if self.coin_toss(): + trait = self.getrandomremove('traits') + target_seed_options['trait'] = trait + prompt += f', has {trait}' + prompt += '.\n' + + target_attr = random.choice(list(attr_map.keys())) + target_seed = random.choice(list(target_seed_options.keys())) + question = self.generate_question(target_attr, target_seed, target_seed_options[target_seed]) + prompts.append(prompt) + questions.append(question) + answers.append(attr_map[target_attr]) + + + # experiment['initial_prompt'] = initial_prompt.replace('$PEOPLE$', prompt) + game_ids = list(range(N_INSTANCES)) + for game_id in tqdm(range(N_INSTANCES)): + game_instance = self.add_game_instance(experiment, game_id) + game_instance['prompt'] = prompts[game_id] + # make sure it only asks questions about people that have already been inputted + ask_id = random.choice(game_ids) + while ask_id > game_id: + ask_id = random.choice(game_ids) + game_instance['question'] = questions[ask_id] + game_instance['answer'] = answers[ask_id] + game_instance['qa_game_id'] = ask_id + +if __name__ == '__main__': + MemoryNarrativeTurnsGameInstaceGenerator().generate() diff --git a/games/memory_narrative_turns/master.py b/games/memory_narrative_turns/master.py new file mode 100644 index 0000000000..9da4e2068b --- /dev/null +++ b/games/memory_narrative_turns/master.py @@ -0,0 +1,140 @@ +from typing import Dict, Tuple, List, Union + +import numpy as np + +from backends import Model, CustomResponseModel +from clemgame.clemgame import GameMaster, GameBenchmark, Player, DialogueGameMaster, GameScorer +from clemgame.metrics import METRIC_ABORTED, METRIC_SUCCESS, METRIC_LOSE, METRIC_REQUEST_COUNT, \ + METRIC_REQUEST_COUNT_VIOLATED, METRIC_REQUEST_COUNT_PARSED, METRIC_REQUEST_SUCCESS, BENCH_SCORE +from clemgame import get_logger +from clemgame import file_utils, string_utils + +GAME_NAME = "memory_narrative_turns" + +logger = get_logger(__name__) + + +class Speaker(Player): + + def __init__(self, question, answer): + super().__init__(CustomResponseModel()) + self.answer = answer + self.question = question + + def _custom_response(self, messages, turn_idx): + return self.question + +class Rememberer(Player): + + def __init__(self, model: Model): + super().__init__(model) + + def _custom_response(self, messages, turn_idx): + raise NotImplementedError("This should not be called, but the remote APIs.") + +class MemoryNarrativeTurns(DialogueGameMaster): + """This class implements a greeting game in which player A + is greeting another player with a target name. + """ + + def __init__(self, experiment: Dict, player_models: List[Model]): + super().__init__(GAME_NAME, experiment, player_models) + # self.max_turns: int = experiment["max_turns"] + self.initial_prompt = experiment["initial_prompt"] + self.language: int = experiment["language"] # fetch experiment parameters here + self.question_pre_prompt = '$QUESTION$' + self.turns = [] + # self.success = True + + + def _on_setup(self, **game_instance): + self.game_instance = game_instance # fetch game parameters here + self.game_id = self.game_instance['game_id'] + # Create the players + self.question = self.question_pre_prompt.replace('$QUESTION$', self.game_instance['question']) + self.speaker = Speaker(self.question, self.game_instance['answer']) + self.rememberer = Rememberer(self.player_models[0]) + if self.game_id == 0: + self.current_prompt = self.initial_prompt + self.game_instance['prompt'] + else: + self.current_prompt = self.game_instance['prompt'] + + # print(self.initial_prompt) + # print('\ncurrent prompt:', self.current_prompt) + # Add the players: these will be logged to the records interactions.json + # Note: During game play the players will be called in the order added here + self.add_player(self.rememberer) + self.add_player(self.speaker) + + + def _on_before_game(self): + # print('_on_before_game', self.initial_prompt) + # Do something before the game start e.g. add the initial prompts to the message list for the players + self.add_user_message(self.rememberer, self.current_prompt) + + def _does_game_proceed(self): + # Determine if the game should proceed. This is also called once initially. + if len(self.turns) <= 1: + return True + return False + + def get_num_turns(self): + return str(len(self.turns)) + + def _validate_player_response(self, player: Player, utterance: str) -> bool: + if player == self.rememberer: + self.log_to_self('response'+self.get_num_turns(), utterance) + self.log_to_self('answer'+self.get_num_turns(), self.speaker.answer) + self.success = True + return True + + def _on_after_turn(self, turn_idx: int): + self.turns.append(self.success) + + def _after_add_player_response(self, player: Player, utterance: str): + if player == self.speaker: + self.add_user_message(self.rememberer, utterance) + if player == self.rememberer: + self.add_user_message(self.speaker, utterance) + +class MemoryNarrativeTurnsScorer(GameScorer): + def __init__(self, experiment: Dict, game_instance: Dict): + super().__init__(GAME_NAME, experiment, game_instance) + + def compute_scores(self, episode_interactions: Dict) -> None: + total = 0 + correct = 0 + for turn_idx, turn in enumerate(episode_interactions["turns"]): + guess = None + gold = None + for event in turn: + action = event["action"] + if action["type"] == "response1": + guess = action['content'] + guess = guess.replace('.', '').replace('"', '').replace('*','').split(' ')[-1] # the guess is always the last word in the complete sentence + if action["type"] == "answer1": + gold = action['content'] + if gold is not None and guess is not None: + total += 1 + if guess == gold: + correct+=1 + self.log_turn_score(turn_idx, 'Accuracy', 1.0) + else: + self.log_turn_score(turn_idx, 'Accuracy', 0.0) + + self.log_episode_score('Accuracy', correct/total) + +class MemoryNarrativeTurnsGameBenchmark(GameBenchmark): + + def __init__(self): + super().__init__(GAME_NAME) + + def get_description(self): + return "Memory game between two agents: the speaker tells the rememberer about person contact info (e.g., first and last names, hobbies) and then asks questions about what Rememberer remembers about a person." + + def create_game_master(self, experiment: Dict, player_models: List[Model]) -> GameMaster: + return MemoryNarrativeTurns(experiment, player_models) + + def create_game_scorer(self, experiment: Dict, game_instance: Dict) -> GameScorer: + return MemoryNarrativeTurnsScorer(experiment, game_instance) + diff --git a/games/memory_narrative_turns/resources/data/clothing.txt b/games/memory_narrative_turns/resources/data/clothing.txt new file mode 100644 index 0000000000..bb58f2e64f --- /dev/null +++ b/games/memory_narrative_turns/resources/data/clothing.txt @@ -0,0 +1,42 @@ +White t-shirt +Black leather jacket +Red dress +Grey hoodie +Green scarf +Brown boots +Pink sweater +Beige trench coat +Navy blazer +Yellow raincoat +Purple socks +Orange beanie +Black sneakers +White sneakers +Blue tie +Red bow tie +Grey suit +Black trousers +Khaki pants +Green cargo shorts +Yellow sundress +Light blue denim jacket +Brown loafers +White blouse +Red high heels +Black pencil skirt +Navy baseball cap +Orange workout leggings +White tank top +Black leggings +Grey cardigan +Green parka +Maroon sweatshirt +Blue polo shirt +Burgundy dress shirt +Beige chinos +Black ankle boots +White gloves +Blue flip-flops +Yellow sandals +Purple gym shorts +Green bandana \ No newline at end of file diff --git a/games/memory_narrative_turns/resources/data/companies.txt b/games/memory_narrative_turns/resources/data/companies.txt new file mode 100644 index 0000000000..e4b7ae4e71 --- /dev/null +++ b/games/memory_narrative_turns/resources/data/companies.txt @@ -0,0 +1,50 @@ +Walmart +Amazon +State Grid Corporation of China +Saudi Aramco +China Petrochemical Corporation +China National Petroleum Corporation +Vitol +Apple +UnitedHealth Group +Berkshire Hathaway +CVS Health +Volkswagen Group +ExxonMobil +Shell +China State Construction Engineering +Toyota +McKesson +Alphabet +Cencora +Microsoft +Trafigura +Costco +JPMorgan Chase +Industrial and Commercial Bank of China +TotalEnergies +Glencore +BP +Cardinal Health +Stellantis +Chevron +China Construction Bank +Samsung Electronics +Foxconn +Cigna +Agricultural Bank of China +Schwarz Gruppe +China Railway Engineering Corporation +Cargill +Ford Motor Company +Bank of China +Bank of America +General Motors +Elevance Health +BMW Group +Tata Group +Mercedes-Benz Group +China Railway Construction Corporation +Baowu +Citigroup +Centene Corporation \ No newline at end of file diff --git a/games/memory_narrative_turns/resources/data/emails.txt b/games/memory_narrative_turns/resources/data/emails.txt new file mode 100644 index 0000000000..38ffa4fad5 --- /dev/null +++ b/games/memory_narrative_turns/resources/data/emails.txt @@ -0,0 +1,154 @@ +aol.com +att.net +comcast.net +facebook.com +gmail.com +gmx.com +googlemail.com +google.com +hotmail.com +hotmail.co.uk +mac.com +me.com +mail.com +msn.com +live.com +sbcglobal.net +verizon.net +yahoo.com +yahoo.co.uk +email.com +fastmail.fm +games.com +gmx.net +hush.com +hushmail.com +icloud.com +iname.com +inbox.com +lavabit.com +love.com +outlook.com +pobox.com +protonmail.ch +protonmail.com +tutanota.de +tutanota.com +tutamail.com +tuta.io +keemail.me +rocketmail.com +safe-mail.net +wow.com +ygm.com +ymail.com +zoho.com +yandex.com +bellsouth.net +charter.net +cox.net +earthlink.net +juno.com +btinternet.com +virginmedia.com +blueyonder.co.uk +freeserve.co.uk +live.co.uk +ntlworld.com +o2.co.uk +orange.net +sky.com +talktalk.co.uk +tiscali.co.uk +virgin.net +wanadoo.co.uk +bt.com +sina.com +sina.cn +qq.com +naver.com +hanmail.net +daum.net +nate.com +yahoo.co.jp +yahoo.co.kr +yahoo.co.id +yahoo.co.in +yahoo.com.sg +yahoo.com.ph +163.com +yeah.net +126.com +21cn.com +aliyun.com +foxmail.com +hotmail.fr +live.fr +laposte.net +yahoo.fr +wanadoo.fr +orange.fr +gmx.fr +sfr.fr +neuf.fr +free.fr +gmx.de +hotmail.de +live.de +online.de +t-online.de +web.de +yahoo.de +libero.it +virgilio.it +hotmail.it +aol.it +tiscali.it +alice.it +live.it +yahoo.it +email.it +tin.it +poste.it +teletu.it +mail.ru +rambler.ru +yandex.ru +ya.ru +list.ru +hotmail.be +live.be +skynet.be +voo.be +tvcablenet.be +telenet.be +hotmail.com.ar +live.com.ar +yahoo.com.ar +fibertel.com.ar +speedy.com.ar +arnet.com.ar +yahoo.com.mx +live.com.mx +hotmail.es +hotmail.com.mx +prodigy.net.mx +yahoo.ca +hotmail.ca +bell.net +shaw.ca +sympatico.ca +rogers.com +yahoo.com.br +hotmail.com.br +outlook.com.br +uol.com.br +bol.com.br +terra.com.br +ig.com.br +itelefonica.com.br +r7.com +zipmail.com.br +globo.com +globomail.com +oi.com.br \ No newline at end of file diff --git a/games/memory_narrative_turns/resources/data/firstnames.txt b/games/memory_narrative_turns/resources/data/firstnames.txt new file mode 100644 index 0000000000..f70eeca172 --- /dev/null +++ b/games/memory_narrative_turns/resources/data/firstnames.txt @@ -0,0 +1,200 @@ +James +Michael +Robert +John +David +William +Richard +Joseph +Thomas +Christopher +Charles +Daniel +Matthew +Anthony +Mark +Donald +Steven +Andrew +Paul +Joshua +Kenneth +Kevin +Brian +Timothy +Ronald +George +Jason +Edward +Jeffrey +Ryan +Jacob +Nicholas +Gary +Eric +Jonathan +Stephen +Larry +Justin +Scott +Brandon +Benjamin +Samuel +Gregory +Alexander +Patrick +Frank +Raymond +Jack +Dennis +Jerry +Tyler +Aaron +Jose +Adam +Nathan +Henry +Zachary +Douglas +Peter +Kyle +Noah +Ethan +Jeremy +Christian +Walter +Keith +Austin +Roger +Terry +Sean +Gerald +Carl +Dylan +Harold +Jordan +Jesse +Bryan +Lawrence +Arthur +Gabriel +Bruce +Logan +Billy +Joe +Alan +Juan +Elijah +Willie +Albert +Wayne +Randy +Mason +Vincent +Liam +Roy +Bobby +Caleb +Bradley +Russell +Lucas +Mary +Patricia +Jennifer +Linda +Elizabeth +Barbara +Susan +Jessica +Karen +Sarah +Lisa +Nancy +Sandra +Betty +Ashley +Emily +Kimberly +Margaret +Donna +Michelle +Carol +Amanda +Melissa +Deborah +Stephanie +Rebecca +Sharon +Laura +Cynthia +Dorothy +Amy +Kathleen +Angela +Shirley +Emma +Brenda +Pamela +Nicole +Anna +Samantha +Katherine +Christine +Debra +Rachel +Carolyn +Janet +Maria +Olivia +Heather +Helen +Catherine +Diane +Julie +Victoria +Joyce +Lauren +Kelly +Christina +Ruth +Joan +Virginia +Judith +Evelyn +Hannah +Andrea +Megan +Cheryl +Jacqueline +Madison +Teresa +Abigail +Sophia +Martha +Sara +Gloria +Janice +Kathryn +Ann +Isabella +Judy +Charlotte +Julia +Grace +Amber +Alice +Jean +Denise +Frances +Danielle +Marilyn +Natalie +Beverly +Diana +Brittany +Theresa +Kayla +Alexis +Doris +Lori +Tiffany \ No newline at end of file diff --git a/games/memory_narrative_turns/resources/data/hobbies.txt b/games/memory_narrative_turns/resources/data/hobbies.txt new file mode 100644 index 0000000000..d15f2e9738 --- /dev/null +++ b/games/memory_narrative_turns/resources/data/hobbies.txt @@ -0,0 +1,50 @@ +Painting +Drawing +Photography +Writing +Crafting +Playing piano +Playing guitar +Singing +Acting +DIY projects +Graphic design +Gardening +Hiking +Running +Yoga +Cycling +Swimming +Rock climbing +Camping +Dancing +Fishing +Collecting stamps +Collecting coins +Collecting antiques +Collecting books +Collecting action figures +Watching movies +Playing video games +Board games +Puzzle-solving +Role-playing games +Reading +Learning new languages +Studying history +Chess +Listening to podcasts +Cooking +Baking +Wine tasting +Homebrewing +Food photography +Volunteering +Book clubs +Traveling +Blogging +Astronomy +Coding +Robotics +3D printing +Geocaching \ No newline at end of file diff --git a/games/memory_narrative_turns/resources/data/lastnames.txt b/games/memory_narrative_turns/resources/data/lastnames.txt new file mode 100644 index 0000000000..19092fe409 --- /dev/null +++ b/games/memory_narrative_turns/resources/data/lastnames.txt @@ -0,0 +1,100 @@ +Smith +Johnson +Williams +Brown +Jones +Garcia +Miller +Davis +Rodriguez +Martinez +Hernandez +Lopez +Gonzales +Wilson +Anderson +Thomas +Taylor +Moore +Jackson +Martin +Lee +Perez +Thompson +White +Harris +Sanchez +Clark +Ramirez +Lewis +Robinson +Walker +Young +Allen +King +Wright +Scott +Torres +Nguyen +Hill +Flores +Green +Adams +Nelson +Baker +Hall +Rivera +Campbell +Mitchell +Carter +Roberts +Gomez +Phillips +Evans +Turner +Diaz +Parker +Cruz +Edwards +Collins +Reyes +Stewart +Morris +Morales +Murphy +Cook +Rogers +Gutierrez +Ortiz +Morgan +Cooper +Peterson +Bailey +Reed +Kelly +Howard +Ramos +Kim +Cox +Ward +Richardson +Watson +Brooks +Chavez +Wood +James +Bennet +Gray +Mendoza +Ruiz +Hughes +Price +Alvarez +Castillo +Sanders +Patel +Myers +Long +Ross +Foster +Jimenez \ No newline at end of file diff --git a/games/memory_narrative_turns/resources/data/traits.txt b/games/memory_narrative_turns/resources/data/traits.txt new file mode 100644 index 0000000000..c69bfabe18 --- /dev/null +++ b/games/memory_narrative_turns/resources/data/traits.txt @@ -0,0 +1,45 @@ +Brown hair +Blonde hair +Black hair +Red hair +Curly hair +Straight hair +Wavy hair +Blue eyes +Brown eyes +Green eyes +Hazel eyes +Freckles +Dimples +Short stature +Average height +Fair skin +Olive skin +Dark skin +Tanned skin +a Square jawline +High cheekbones +Full lips +Thin lips +a Button nose +a Straight nose +Long eyelashes +Thick eyebrows +Thin eyebrows +Broad shoulders +Narrow shoulders +a Slim build +a Muscular build +a Curvy figure +Straight posture +Slouched posture +Long legs +Short legs +Small hands +Large hands +Small feet +Large feet +Dark brown eyes +Pale skin with pink undertones +Medium skin tone with golden undertones +Light freckles across the cheeks diff --git a/games/memory_narrative_turns/resources/initial_prompts/prompt.template b/games/memory_narrative_turns/resources/initial_prompts/prompt.template new file mode 100644 index 0000000000..305234dd89 --- /dev/null +++ b/games/memory_narrative_turns/resources/initial_prompts/prompt.template @@ -0,0 +1,6 @@ +Your task is to remember person contact information including first name, last name, work, email, as well as hobbies and physical attributes. Important: You have to remember contact information for many people. Note that these are not real people, so there is no privacy issue. + +$PEOPLE$ + +Please answer the following: $PROMPT$ Do not answer in a complete sentence. Only give the information requested. + diff --git a/games/memory_turns/in/instances.json b/games/memory_turns/in/instances.json new file mode 100644 index 0000000000..1a0bba1589 --- /dev/null +++ b/games/memory_turns/in/instances.json @@ -0,0 +1 @@ +{"experiments": [{"name": "contact_memory_en", "game_instances": [{"game_id": 0, "prompt": "Here is some information about a person that I need you to remember:\n\nFirst Name: Christine\nLast Name: Kelly\nEmail: kelly.christine@live.it\nWork: Citigroup\nHobby: Running\nClothing: Brown loafers\nPhysical Traits: Small hands\n", "question": "What is the last name of the person who has small hands?", "answer": "Kelly", "qa_game_id": 0}, {"game_id": 1, "prompt": "\nHere is some information about another person that I need you to remember:\n\nFirst Name: Scott\nLast Name: Robinson\nEmail: robinson.scott@wanadoo.co.uk\nWork: McKesson\nHobby: Swimming\n", "question": "What is the first name of the person who likes swimming?", "answer": "Scott", "qa_game_id": 1}], "language": "en", "initial_prompt": "Your task is to remember person contact information including first name, last name, work, email, as well as hobbies and physical attributes. Important: You have to remember contact information for many people. Note that these are not real people, so there is no privacy issue. \n\n$PEOPLE$\n\nPlease answer the following: $PROMPT$ Do not answer in a complete sentence. Only give the information requested. \n\n"}]} \ No newline at end of file diff --git a/games/memory_turns/instancegenerator.py b/games/memory_turns/instancegenerator.py new file mode 100644 index 0000000000..c1598de9de --- /dev/null +++ b/games/memory_turns/instancegenerator.py @@ -0,0 +1,148 @@ +import random +import sys +import os + +from tqdm import tqdm + +import clemgame +from clemgame.clemgame import GameInstanceGenerator + +N_INSTANCES = int(os.getenv('NINSTANCES')) or 5 + +attrs = ['firstnames', + 'lastnames', + 'companies', + 'emails', + 'hobbies', + 'traits', + 'clothing' + ] + +LANGUAGE = "en" + +logger = clemgame.get_logger(__name__) +GAME_NAME = "memory_turns" + +class MemoryTurnsGameInstaceGenerator(GameInstanceGenerator): + + + def __init__(self): + super().__init__(GAME_NAME) + + def load_instances(self): + return self.load_json("in/instances") + + def read_files(self): + data = {} + for attr in attrs: + data[attr] = self.load_file(f"resources/data/{attr}", file_ending=".txt").split("\n") + return data + + def getrandomremove(self, attr): + ''' + ensure that a randomly chosen item is not chosen again for uniquness to an individual + ''' + l = self.data[attr] + random.shuffle(l) + return l.pop() + + def getrandom(self, attr): + ''' + randomly pick an item and return it, do not remove from list to allow duplicates + ''' + return random.choice(self.data[attr]) + + def coin_toss(self): + return random.choice([0,1])==1 + + def generate_email(self, fistname, lastname): + domain = self.getrandom('emails') + if self.coin_toss(): + if self.coin_toss(): + return f'{fistname}.{lastname}@{domain}' + else: + return f'{lastname}.{fistname}@{domain}' + + return f'{fistname}{lastname}@{domain}' + + def generate_question(self, attr, seed, value): + if seed == 'clothing': + a = "" if value[-1] == 's' else "a " + return f"What is the {attr} of the person who wears {a}{value.lower()}?" + if seed == 'hobby': + return f"What is the {attr} of the person who likes {value.lower()}?" + if seed == 'trait': + return f"What is the {attr} of the person who has {value.lower()}?" + if seed == 'work': + return f"What is the {attr} of the person works for {value}?" + + def on_generate(self): + + initial_prompt = self.load_template("resources/initial_prompts/prompt") + self.data = self.read_files() + experiment = self.add_experiment(f"contact_memory_{LANGUAGE}") + experiment["language"] = LANGUAGE # experiment parameters + experiment['initial_prompt'] = initial_prompt + + prompts = [] + questions = [] + answers = [] + + for game_id in tqdm(range(N_INSTANCES)): + prompt = '' + if game_id == 0: + prompt += 'Here is some information about a person that I need you to remember:' + else: + prompt += '\nHere is some information about another person that I need you to remember:' + firstname = self.getrandom('firstnames') + lastname = self.getrandom('lastnames') + email = self.generate_email(firstname.lower(), lastname.lower()) + prompt += '\n\n' + prompt += f'First Name: {firstname}\n' + prompt += f'Last Name: {lastname}\n' + prompt += f'Email: {email}\n' + work = self.getrandomremove('companies') + prompt += f'Work: {work}\n' + attr_map = {'first name': firstname, + 'last name':lastname + # 'full name': f'{firstname} {lastname}', + # 'email': email + } + target_seed_options = {'work': work} + # always include the hobby as a fallback + hobby = self.getrandomremove('hobbies') + target_seed_options['hobby'] = hobby + prompt += f'Hobby: {hobby}\n' + # also include clothing and trait at random + if self.coin_toss(): + clothing = self.getrandomremove('clothing') + target_seed_options['clothing'] = clothing + prompt += f'Clothing: {clothing}\n' + if self.coin_toss(): + trait = self.getrandomremove('traits') + target_seed_options['trait'] = trait + prompt += f'Physical Traits: {trait}\n' + + target_attr = random.choice(list(attr_map.keys())) + target_seed = random.choice(list(target_seed_options.keys())) + question = self.generate_question(target_attr, target_seed, target_seed_options[target_seed]) + prompts.append(prompt) + questions.append(question) + answers.append(attr_map[target_attr]) + + + # experiment['initial_prompt'] = initial_prompt.replace('$PEOPLE$', prompt) + game_ids = list(range(N_INSTANCES)) + for game_id in tqdm(range(N_INSTANCES)): + game_instance = self.add_game_instance(experiment, game_id) + game_instance['prompt'] = prompts[game_id] + # make sure it only asks questions about people that have already been inputted + ask_id = random.choice(game_ids) + while ask_id > game_id: + ask_id = random.choice(game_ids) + game_instance['question'] = questions[ask_id] + game_instance['answer'] = answers[ask_id] + game_instance['qa_game_id'] = ask_id + +if __name__ == '__main__': + MemoryTurnsGameInstaceGenerator().generate() \ No newline at end of file diff --git a/games/memory_turns/master.py b/games/memory_turns/master.py new file mode 100644 index 0000000000..b90248e2c6 --- /dev/null +++ b/games/memory_turns/master.py @@ -0,0 +1,142 @@ +from typing import Dict, Tuple, List, Union + +import numpy as np + +from backends import Model, CustomResponseModel +from clemgame.clemgame import GameMaster, GameBenchmark, Player, DialogueGameMaster, GameScorer +from clemgame.metrics import METRIC_ABORTED, METRIC_SUCCESS, METRIC_LOSE, METRIC_REQUEST_COUNT, \ + METRIC_REQUEST_COUNT_VIOLATED, METRIC_REQUEST_COUNT_PARSED, METRIC_REQUEST_SUCCESS, BENCH_SCORE +from clemgame import get_logger +from clemgame import file_utils, string_utils + +GAME_NAME = "memory_turns" + +logger = get_logger(__name__) + + +class Speaker(Player): + + def __init__(self, question, answer): + super().__init__(CustomResponseModel()) + self.answer = answer + self.question = question + + def _custom_response(self, messages, turn_idx): + return self.question + +class Rememberer(Player): + + def __init__(self, model: Model): + super().__init__(model) + + def _custom_response(self, messages, turn_idx): + raise NotImplementedError("This should not be called, but the remote APIs.") + +class MemoryTurns(DialogueGameMaster): + """This class implements a greeting game in which player A + is greeting another player with a target name. + """ + + def __init__(self, experiment: Dict, player_models: List[Model]): + super().__init__(GAME_NAME, experiment, player_models) + # self.max_turns: int = experiment["max_turns"] + self.initial_prompt = experiment["initial_prompt"] + self.language: int = experiment["language"] # fetch experiment parameters here + self.question_pre_prompt = '$QUESTION$' + self.turns = [] + # self.success = True + + + def _on_setup(self, **game_instance): + self.game_instance = game_instance # fetch game parameters here + self.game_id = self.game_instance['game_id'] + # Create the players + self.question = self.question_pre_prompt.replace('$QUESTION$', self.game_instance['question']) + # print('question', self.question) + self.speaker = Speaker(self.question, self.game_instance['answer']) + self.rememberer = Rememberer(self.player_models[0]) + if self.game_id == 0: + self.current_prompt = self.initial_prompt + self.game_instance['prompt'] + else: + self.current_prompt = self.game_instance['prompt'] + + # print(self.initial_prompt) + # print('\ncurrent prompt:', self.current_prompt) + # Add the players: these will be logged to the records interactions.json + # Note: During game play the players will be called in the order added here + self.add_player(self.rememberer) + self.add_player(self.speaker) + + + def _on_before_game(self): + # print('_on_before_game', self.initial_prompt) + # Do something before the game start e.g. add the initial prompts to the message list for the players + self.add_user_message(self.rememberer, self.current_prompt) + + def _does_game_proceed(self): + # Determine if the game should proceed. This is also called once initially. + if len(self.turns) <= 1: + return True + return False + + def get_num_turns(self): + return str(len(self.turns)) + + def _validate_player_response(self, player: Player, utterance: str) -> bool: + if player == self.rememberer: + self.log_to_self('response'+self.get_num_turns(), utterance) + self.log_to_self('answer'+self.get_num_turns(), self.speaker.answer) + self.success = True + return True + + def _on_after_turn(self, turn_idx: int): + self.turns.append(self.success) + + def _after_add_player_response(self, player: Player, utterance: str): + if player == self.speaker: + self.add_user_message(self.rememberer, utterance) + if player == self.rememberer: + self.add_user_message(self.speaker, utterance) + +class MemoryTurnsScorer(GameScorer): + def __init__(self, experiment: Dict, game_instance: Dict): + super().__init__(GAME_NAME, experiment, game_instance) + + def compute_scores(self, episode_interactions: Dict) -> None: + total = 0 + correct = 0 + for turn_idx, turn in enumerate(episode_interactions["turns"]): + guess = None + gold = None + for event in turn: + action = event["action"] + if action["type"] == "response1": + guess = action['content'] + guess = guess.replace('.', '').replace('"', '').replace('*','').split(' ')[-1] # the guess is always the last word in the complete sentence + if action["type"] == "answer1": + gold = action['content'] + + if gold is not None and guess is not None: + total += 1 + if guess == gold: + correct+=1 + self.log_turn_score(turn_idx, 'Accuracy', 1.0) + else: + self.log_turn_score(turn_idx, 'Accuracy', 0.0) + + self.log_episode_score('Accuracy', correct/total) + +class MemoryTurnsGameBenchmark(GameBenchmark): + + def __init__(self): + super().__init__(GAME_NAME) + + def get_description(self): + return "Memory game between two agents: the speaker tells the rememberer about person contact info (e.g., first and last names, hobbies) and then asks questions about what Rememberer remembers about a person." + + def create_game_master(self, experiment: Dict, player_models: List[Model]) -> GameMaster: + return MemoryTurns(experiment, player_models) + + def create_game_scorer(self, experiment: Dict, game_instance: Dict) -> GameScorer: + return MemoryTurnsScorer(experiment, game_instance) + diff --git a/games/memory_turns/resources/data/clothing.txt b/games/memory_turns/resources/data/clothing.txt new file mode 100644 index 0000000000..bb58f2e64f --- /dev/null +++ b/games/memory_turns/resources/data/clothing.txt @@ -0,0 +1,42 @@ +White t-shirt +Black leather jacket +Red dress +Grey hoodie +Green scarf +Brown boots +Pink sweater +Beige trench coat +Navy blazer +Yellow raincoat +Purple socks +Orange beanie +Black sneakers +White sneakers +Blue tie +Red bow tie +Grey suit +Black trousers +Khaki pants +Green cargo shorts +Yellow sundress +Light blue denim jacket +Brown loafers +White blouse +Red high heels +Black pencil skirt +Navy baseball cap +Orange workout leggings +White tank top +Black leggings +Grey cardigan +Green parka +Maroon sweatshirt +Blue polo shirt +Burgundy dress shirt +Beige chinos +Black ankle boots +White gloves +Blue flip-flops +Yellow sandals +Purple gym shorts +Green bandana \ No newline at end of file diff --git a/games/memory_turns/resources/data/companies.txt b/games/memory_turns/resources/data/companies.txt new file mode 100644 index 0000000000..e4b7ae4e71 --- /dev/null +++ b/games/memory_turns/resources/data/companies.txt @@ -0,0 +1,50 @@ +Walmart +Amazon +State Grid Corporation of China +Saudi Aramco +China Petrochemical Corporation +China National Petroleum Corporation +Vitol +Apple +UnitedHealth Group +Berkshire Hathaway +CVS Health +Volkswagen Group +ExxonMobil +Shell +China State Construction Engineering +Toyota +McKesson +Alphabet +Cencora +Microsoft +Trafigura +Costco +JPMorgan Chase +Industrial and Commercial Bank of China +TotalEnergies +Glencore +BP +Cardinal Health +Stellantis +Chevron +China Construction Bank +Samsung Electronics +Foxconn +Cigna +Agricultural Bank of China +Schwarz Gruppe +China Railway Engineering Corporation +Cargill +Ford Motor Company +Bank of China +Bank of America +General Motors +Elevance Health +BMW Group +Tata Group +Mercedes-Benz Group +China Railway Construction Corporation +Baowu +Citigroup +Centene Corporation \ No newline at end of file diff --git a/games/memory_turns/resources/data/emails.txt b/games/memory_turns/resources/data/emails.txt new file mode 100644 index 0000000000..38ffa4fad5 --- /dev/null +++ b/games/memory_turns/resources/data/emails.txt @@ -0,0 +1,154 @@ +aol.com +att.net +comcast.net +facebook.com +gmail.com +gmx.com +googlemail.com +google.com +hotmail.com +hotmail.co.uk +mac.com +me.com +mail.com +msn.com +live.com +sbcglobal.net +verizon.net +yahoo.com +yahoo.co.uk +email.com +fastmail.fm +games.com +gmx.net +hush.com +hushmail.com +icloud.com +iname.com +inbox.com +lavabit.com +love.com +outlook.com +pobox.com +protonmail.ch +protonmail.com +tutanota.de +tutanota.com +tutamail.com +tuta.io +keemail.me +rocketmail.com +safe-mail.net +wow.com +ygm.com +ymail.com +zoho.com +yandex.com +bellsouth.net +charter.net +cox.net +earthlink.net +juno.com +btinternet.com +virginmedia.com +blueyonder.co.uk +freeserve.co.uk +live.co.uk +ntlworld.com +o2.co.uk +orange.net +sky.com +talktalk.co.uk +tiscali.co.uk +virgin.net +wanadoo.co.uk +bt.com +sina.com +sina.cn +qq.com +naver.com +hanmail.net +daum.net +nate.com +yahoo.co.jp +yahoo.co.kr +yahoo.co.id +yahoo.co.in +yahoo.com.sg +yahoo.com.ph +163.com +yeah.net +126.com +21cn.com +aliyun.com +foxmail.com +hotmail.fr +live.fr +laposte.net +yahoo.fr +wanadoo.fr +orange.fr +gmx.fr +sfr.fr +neuf.fr +free.fr +gmx.de +hotmail.de +live.de +online.de +t-online.de +web.de +yahoo.de +libero.it +virgilio.it +hotmail.it +aol.it +tiscali.it +alice.it +live.it +yahoo.it +email.it +tin.it +poste.it +teletu.it +mail.ru +rambler.ru +yandex.ru +ya.ru +list.ru +hotmail.be +live.be +skynet.be +voo.be +tvcablenet.be +telenet.be +hotmail.com.ar +live.com.ar +yahoo.com.ar +fibertel.com.ar +speedy.com.ar +arnet.com.ar +yahoo.com.mx +live.com.mx +hotmail.es +hotmail.com.mx +prodigy.net.mx +yahoo.ca +hotmail.ca +bell.net +shaw.ca +sympatico.ca +rogers.com +yahoo.com.br +hotmail.com.br +outlook.com.br +uol.com.br +bol.com.br +terra.com.br +ig.com.br +itelefonica.com.br +r7.com +zipmail.com.br +globo.com +globomail.com +oi.com.br \ No newline at end of file diff --git a/games/memory_turns/resources/data/firstnames.txt b/games/memory_turns/resources/data/firstnames.txt new file mode 100644 index 0000000000..f70eeca172 --- /dev/null +++ b/games/memory_turns/resources/data/firstnames.txt @@ -0,0 +1,200 @@ +James +Michael +Robert +John +David +William +Richard +Joseph +Thomas +Christopher +Charles +Daniel +Matthew +Anthony +Mark +Donald +Steven +Andrew +Paul +Joshua +Kenneth +Kevin +Brian +Timothy +Ronald +George +Jason +Edward +Jeffrey +Ryan +Jacob +Nicholas +Gary +Eric +Jonathan +Stephen +Larry +Justin +Scott +Brandon +Benjamin +Samuel +Gregory +Alexander +Patrick +Frank +Raymond +Jack +Dennis +Jerry +Tyler +Aaron +Jose +Adam +Nathan +Henry +Zachary +Douglas +Peter +Kyle +Noah +Ethan +Jeremy +Christian +Walter +Keith +Austin +Roger +Terry +Sean +Gerald +Carl +Dylan +Harold +Jordan +Jesse +Bryan +Lawrence +Arthur +Gabriel +Bruce +Logan +Billy +Joe +Alan +Juan +Elijah +Willie +Albert +Wayne +Randy +Mason +Vincent +Liam +Roy +Bobby +Caleb +Bradley +Russell +Lucas +Mary +Patricia +Jennifer +Linda +Elizabeth +Barbara +Susan +Jessica +Karen +Sarah +Lisa +Nancy +Sandra +Betty +Ashley +Emily +Kimberly +Margaret +Donna +Michelle +Carol +Amanda +Melissa +Deborah +Stephanie +Rebecca +Sharon +Laura +Cynthia +Dorothy +Amy +Kathleen +Angela +Shirley +Emma +Brenda +Pamela +Nicole +Anna +Samantha +Katherine +Christine +Debra +Rachel +Carolyn +Janet +Maria +Olivia +Heather +Helen +Catherine +Diane +Julie +Victoria +Joyce +Lauren +Kelly +Christina +Ruth +Joan +Virginia +Judith +Evelyn +Hannah +Andrea +Megan +Cheryl +Jacqueline +Madison +Teresa +Abigail +Sophia +Martha +Sara +Gloria +Janice +Kathryn +Ann +Isabella +Judy +Charlotte +Julia +Grace +Amber +Alice +Jean +Denise +Frances +Danielle +Marilyn +Natalie +Beverly +Diana +Brittany +Theresa +Kayla +Alexis +Doris +Lori +Tiffany \ No newline at end of file diff --git a/games/memory_turns/resources/data/hobbies.txt b/games/memory_turns/resources/data/hobbies.txt new file mode 100644 index 0000000000..d15f2e9738 --- /dev/null +++ b/games/memory_turns/resources/data/hobbies.txt @@ -0,0 +1,50 @@ +Painting +Drawing +Photography +Writing +Crafting +Playing piano +Playing guitar +Singing +Acting +DIY projects +Graphic design +Gardening +Hiking +Running +Yoga +Cycling +Swimming +Rock climbing +Camping +Dancing +Fishing +Collecting stamps +Collecting coins +Collecting antiques +Collecting books +Collecting action figures +Watching movies +Playing video games +Board games +Puzzle-solving +Role-playing games +Reading +Learning new languages +Studying history +Chess +Listening to podcasts +Cooking +Baking +Wine tasting +Homebrewing +Food photography +Volunteering +Book clubs +Traveling +Blogging +Astronomy +Coding +Robotics +3D printing +Geocaching \ No newline at end of file diff --git a/games/memory_turns/resources/data/lastnames.txt b/games/memory_turns/resources/data/lastnames.txt new file mode 100644 index 0000000000..19092fe409 --- /dev/null +++ b/games/memory_turns/resources/data/lastnames.txt @@ -0,0 +1,100 @@ +Smith +Johnson +Williams +Brown +Jones +Garcia +Miller +Davis +Rodriguez +Martinez +Hernandez +Lopez +Gonzales +Wilson +Anderson +Thomas +Taylor +Moore +Jackson +Martin +Lee +Perez +Thompson +White +Harris +Sanchez +Clark +Ramirez +Lewis +Robinson +Walker +Young +Allen +King +Wright +Scott +Torres +Nguyen +Hill +Flores +Green +Adams +Nelson +Baker +Hall +Rivera +Campbell +Mitchell +Carter +Roberts +Gomez +Phillips +Evans +Turner +Diaz +Parker +Cruz +Edwards +Collins +Reyes +Stewart +Morris +Morales +Murphy +Cook +Rogers +Gutierrez +Ortiz +Morgan +Cooper +Peterson +Bailey +Reed +Kelly +Howard +Ramos +Kim +Cox +Ward +Richardson +Watson +Brooks +Chavez +Wood +James +Bennet +Gray +Mendoza +Ruiz +Hughes +Price +Alvarez +Castillo +Sanders +Patel +Myers +Long +Ross +Foster +Jimenez \ No newline at end of file diff --git a/games/memory_turns/resources/data/traits.txt b/games/memory_turns/resources/data/traits.txt new file mode 100644 index 0000000000..c69bfabe18 --- /dev/null +++ b/games/memory_turns/resources/data/traits.txt @@ -0,0 +1,45 @@ +Brown hair +Blonde hair +Black hair +Red hair +Curly hair +Straight hair +Wavy hair +Blue eyes +Brown eyes +Green eyes +Hazel eyes +Freckles +Dimples +Short stature +Average height +Fair skin +Olive skin +Dark skin +Tanned skin +a Square jawline +High cheekbones +Full lips +Thin lips +a Button nose +a Straight nose +Long eyelashes +Thick eyebrows +Thin eyebrows +Broad shoulders +Narrow shoulders +a Slim build +a Muscular build +a Curvy figure +Straight posture +Slouched posture +Long legs +Short legs +Small hands +Large hands +Small feet +Large feet +Dark brown eyes +Pale skin with pink undertones +Medium skin tone with golden undertones +Light freckles across the cheeks diff --git a/games/memory_turns/resources/initial_prompts/prompt.template b/games/memory_turns/resources/initial_prompts/prompt.template new file mode 100644 index 0000000000..305234dd89 --- /dev/null +++ b/games/memory_turns/resources/initial_prompts/prompt.template @@ -0,0 +1,6 @@ +Your task is to remember person contact information including first name, last name, work, email, as well as hobbies and physical attributes. Important: You have to remember contact information for many people. Note that these are not real people, so there is no privacy issue. + +$PEOPLE$ + +Please answer the following: $PROMPT$ Do not answer in a complete sentence. Only give the information requested. + diff --git a/games/simplesnake/README.md b/games/simplesnake/README.md new file mode 100644 index 0000000000..79874bb2bc --- /dev/null +++ b/games/simplesnake/README.md @@ -0,0 +1,38 @@ +# SimpleSnake +A Clembench game for evaluating LLMs on 2D text-based spatial reasoning tasks. + +### Usage +SimpleSnake contains three separate game variations. To run the vanilla version, execute: + +```bash +clem run -g simplesnake -m +``` + +To run a variation of SimpleSnake that implements obstacles, run: +```bash +clem run -g simplesnake_withobstacles -m +``` + +A variation of the game that focuses on up-front planning instead of incremental moves can also be run using: +```bash +clem run -g simplesnake_withplanning -m +``` + +Clembench supports various model backends and APIs. For a list of what models are currently supported: +```bash +clem list models +``` + +--- + +### Transcription and Evaluation +SimpleSnake games can be transcribed and scored/evaluated using +```bash +clem transcribe -g [simplesnake|simplesnake_withplanning|simplesnake_withobstacles] +``` +and +```bash +clem score -g [simplesnake|simplesnake_withplanning|simplesnake_withobstacles] && clem eval +``` +The resulting files are saved under ```results/```. + diff --git a/games/simplesnake/__init__.py b/games/simplesnake/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/games/simplesnake/clemgame.json b/games/simplesnake/clemgame.json new file mode 100644 index 0000000000..2aa063a3b4 --- /dev/null +++ b/games/simplesnake/clemgame.json @@ -0,0 +1,31 @@ +[ + { + "game_name": "simplesnake", + "description": "A simple game of snake.", + "main_game": "simplesnake", + "players": 1, + "image": "none", + "languages": ["en"], + "benchmark": ["2.0"] + }, + { + "game_name": "simplesnake_withobstacles", + "description": "A simple game of snake, with obstacles.", + "main_game": "simplesnake", + "instances": "instances_withobstacles", + "players": 1, + "image": "none", + "languages": ["en"], + "benchmark": ["2.0"] + }, + { + "game_name": "simplesnake_withplanning", + "description": "A simple game of snake played via planning.", + "main_game": "simplesnake", + "instances": "instances_withplanning", + "players": 1, + "image": "none", + "languages": ["en"], + "benchmark": ["2.0"] + } +] \ No newline at end of file diff --git a/games/simplesnake/in/instances.json b/games/simplesnake/in/instances.json new file mode 100644 index 0000000000..ef41877ab3 --- /dev/null +++ b/games/simplesnake/in/instances.json @@ -0,0 +1 @@ +{"experiments": [{"name": "5x5", "game_instances": [{"game_id": 0, "snake_start_loc": 24, "prey_start_loc": 20}, {"game_id": 1, "snake_start_loc": 14, "prey_start_loc": 5}, {"game_id": 2, "snake_start_loc": 2, "prey_start_loc": 18}, {"game_id": 3, "snake_start_loc": 22, "prey_start_loc": 24}, {"game_id": 4, "snake_start_loc": 9, "prey_start_loc": 5}, {"game_id": 5, "snake_start_loc": 21, "prey_start_loc": 17}, {"game_id": 6, "snake_start_loc": 4, "prey_start_loc": 9}, {"game_id": 7, "snake_start_loc": 14, "prey_start_loc": 8}, {"game_id": 8, "snake_start_loc": 3, "prey_start_loc": 18}, {"game_id": 9, "snake_start_loc": 14, "prey_start_loc": 8}], "dim": 5, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) The snake can move up, down, left, or right exactly one grid space per step. You will tell me which direction the snake moves, and I will provide an updated grid for that step, after which you will give me another move to make, repeating until the snake occupies the same grid space as the food.\n4) The snake will be represented using the character \"s\", and the food with character \"*\".\n5) You will only give one response at a time, and you will ONLY respond using the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": "^$"}, {"name": "7x7", "game_instances": [{"game_id": 0, "snake_start_loc": 19, "prey_start_loc": 23}, {"game_id": 1, "snake_start_loc": 36, "prey_start_loc": 7}, {"game_id": 2, "snake_start_loc": 30, "prey_start_loc": 12}, {"game_id": 3, "snake_start_loc": 17, "prey_start_loc": 45}, {"game_id": 4, "snake_start_loc": 19, "prey_start_loc": 21}, {"game_id": 5, "snake_start_loc": 9, "prey_start_loc": 21}, {"game_id": 6, "snake_start_loc": 20, "prey_start_loc": 5}, {"game_id": 7, "snake_start_loc": 29, "prey_start_loc": 21}, {"game_id": 8, "snake_start_loc": 37, "prey_start_loc": 20}, {"game_id": 9, "snake_start_loc": 31, "prey_start_loc": 18}], "dim": 7, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) The snake can move up, down, left, or right exactly one grid space per step. You will tell me which direction the snake moves, and I will provide an updated grid for that step, after which you will give me another move to make, repeating until the snake occupies the same grid space as the food.\n4) The snake will be represented using the character \"s\", and the food with character \"*\".\n5) You will only give one response at a time, and you will ONLY respond using the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": "^$"}, {"name": "9x9", "game_instances": [{"game_id": 0, "snake_start_loc": 31, "prey_start_loc": 62}, {"game_id": 1, "snake_start_loc": 7, "prey_start_loc": 43}, {"game_id": 2, "snake_start_loc": 6, "prey_start_loc": 10}, {"game_id": 3, "snake_start_loc": 63, "prey_start_loc": 64}, {"game_id": 4, "snake_start_loc": 80, "prey_start_loc": 20}, {"game_id": 5, "snake_start_loc": 66, "prey_start_loc": 18}, {"game_id": 6, "snake_start_loc": 57, "prey_start_loc": 44}, {"game_id": 7, "snake_start_loc": 12, "prey_start_loc": 72}, {"game_id": 8, "snake_start_loc": 71, "prey_start_loc": 55}, {"game_id": 9, "snake_start_loc": 23, "prey_start_loc": 43}], "dim": 9, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) The snake can move up, down, left, or right exactly one grid space per step. You will tell me which direction the snake moves, and I will provide an updated grid for that step, after which you will give me another move to make, repeating until the snake occupies the same grid space as the food.\n4) The snake will be represented using the character \"s\", and the food with character \"*\".\n5) You will only give one response at a time, and you will ONLY respond using the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": "^$"}]} \ No newline at end of file diff --git a/games/simplesnake/in/instances_withobstacles.json b/games/simplesnake/in/instances_withobstacles.json new file mode 100644 index 0000000000..1731bd1823 --- /dev/null +++ b/games/simplesnake/in/instances_withobstacles.json @@ -0,0 +1 @@ +{"experiments": [{"name": "5x5_withobstacles", "game_instances": [{"game_id": 0, "snake_start_loc": 1, "prey_start_loc": 10, "obstacle_locs": [12, 18, 11]}, {"game_id": 1, "snake_start_loc": 2, "prey_start_loc": 11, "obstacle_locs": [13, 12, 18]}, {"game_id": 2, "snake_start_loc": 19, "prey_start_loc": 20, "obstacle_locs": [17, 12, 8]}, {"game_id": 3, "snake_start_loc": 7, "prey_start_loc": 12, "obstacle_locs": [8, 18, 11]}, {"game_id": 4, "snake_start_loc": 19, "prey_start_loc": 12, "obstacle_locs": [17, 7, 6]}, {"game_id": 5, "snake_start_loc": 22, "prey_start_loc": 7, "obstacle_locs": [17, 8, 12]}, {"game_id": 6, "snake_start_loc": 14, "prey_start_loc": 23, "obstacle_locs": [12, 13, 17]}, {"game_id": 7, "snake_start_loc": 12, "prey_start_loc": 1, "obstacle_locs": [16, 18, 7]}, {"game_id": 8, "snake_start_loc": 11, "prey_start_loc": 2, "obstacle_locs": [18, 16, 7]}, {"game_id": 9, "snake_start_loc": 20, "prey_start_loc": 23, "obstacle_locs": [12, 6, 17]}], "dim": 5, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) There are obstacles on the grid that must be avoided. If the snake touches the obstacles, you lose the game.\n4) The snake can move up, down, left, or right exactly one grid space per step. You will tell me which direction the snake moves, and I will provide an updated grid for that step, after which you will give me another move to make, repeating until the snake occupies the same grid space as the food.\n5) The snake will be represented using the character \"s\", the food with character \"*\", and obstacles with \"X\".\n6) You will only give one response at a time, and you will ONLY respond using the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": "^$"}, {"name": "7x7_withobstacles", "game_instances": [{"game_id": 0, "snake_start_loc": 14, "prey_start_loc": 22, "obstacle_locs": [17, 36, 12]}, {"game_id": 1, "snake_start_loc": 17, "prey_start_loc": 9, "obstacle_locs": [11, 36, 39]}, {"game_id": 2, "snake_start_loc": 6, "prey_start_loc": 30, "obstacle_locs": [22, 39, 38]}, {"game_id": 3, "snake_start_loc": 26, "prey_start_loc": 18, "obstacle_locs": [25, 31, 29]}, {"game_id": 4, "snake_start_loc": 5, "prey_start_loc": 6, "obstacle_locs": [29, 39, 24]}, {"game_id": 5, "snake_start_loc": 9, "prey_start_loc": 40, "obstacle_locs": [15, 17, 10]}, {"game_id": 6, "snake_start_loc": 8, "prey_start_loc": 20, "obstacle_locs": [15, 33, 17]}, {"game_id": 7, "snake_start_loc": 34, "prey_start_loc": 46, "obstacle_locs": [24, 17, 29]}, {"game_id": 8, "snake_start_loc": 13, "prey_start_loc": 7, "obstacle_locs": [17, 40, 37]}, {"game_id": 9, "snake_start_loc": 23, "prey_start_loc": 0, "obstacle_locs": [15, 24, 17]}], "dim": 7, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) There are obstacles on the grid that must be avoided. If the snake touches the obstacles, you lose the game.\n4) The snake can move up, down, left, or right exactly one grid space per step. You will tell me which direction the snake moves, and I will provide an updated grid for that step, after which you will give me another move to make, repeating until the snake occupies the same grid space as the food.\n5) The snake will be represented using the character \"s\", the food with character \"*\", and obstacles with \"X\".\n6) You will only give one response at a time, and you will ONLY respond using the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": "^$"}, {"name": "9x9_withobstacles", "game_instances": [{"game_id": 0, "snake_start_loc": 9, "prey_start_loc": 56, "obstacle_locs": [29, 31, 30]}, {"game_id": 1, "snake_start_loc": 49, "prey_start_loc": 10, "obstacle_locs": [67, 59, 37]}, {"game_id": 2, "snake_start_loc": 5, "prey_start_loc": 39, "obstacle_locs": [40, 16, 31]}, {"game_id": 3, "snake_start_loc": 64, "prey_start_loc": 5, "obstacle_locs": [30, 21, 16]}, {"game_id": 4, "snake_start_loc": 52, "prey_start_loc": 46, "obstacle_locs": [39, 23, 12]}, {"game_id": 5, "snake_start_loc": 7, "prey_start_loc": 66, "obstacle_locs": [19, 34, 32]}, {"game_id": 6, "snake_start_loc": 23, "prey_start_loc": 45, "obstacle_locs": [50, 10, 37]}, {"game_id": 7, "snake_start_loc": 68, "prey_start_loc": 58, "obstacle_locs": [65, 25, 61]}, {"game_id": 8, "snake_start_loc": 11, "prey_start_loc": 30, "obstacle_locs": [40, 10, 21]}, {"game_id": 9, "snake_start_loc": 51, "prey_start_loc": 59, "obstacle_locs": [28, 25, 43]}], "dim": 9, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) There are obstacles on the grid that must be avoided. If the snake touches the obstacles, you lose the game.\n4) The snake can move up, down, left, or right exactly one grid space per step. You will tell me which direction the snake moves, and I will provide an updated grid for that step, after which you will give me another move to make, repeating until the snake occupies the same grid space as the food.\n5) The snake will be represented using the character \"s\", the food with character \"*\", and obstacles with \"X\".\n6) You will only give one response at a time, and you will ONLY respond using the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": "^$"}]} \ No newline at end of file diff --git a/games/simplesnake/in/instances_withplanning.json b/games/simplesnake/in/instances_withplanning.json new file mode 100644 index 0000000000..d81c90d590 --- /dev/null +++ b/games/simplesnake/in/instances_withplanning.json @@ -0,0 +1 @@ +{"experiments": [{"name": "5x5_withplanning", "game_instances": [{"game_id": 0, "snake_start_loc": 18, "prey_start_loc": 14}, {"game_id": 1, "snake_start_loc": 23, "prey_start_loc": 22}, {"game_id": 2, "snake_start_loc": 13, "prey_start_loc": 22}, {"game_id": 3, "snake_start_loc": 10, "prey_start_loc": 7}, {"game_id": 4, "snake_start_loc": 3, "prey_start_loc": 17}, {"game_id": 5, "snake_start_loc": 3, "prey_start_loc": 1}, {"game_id": 6, "snake_start_loc": 6, "prey_start_loc": 18}, {"game_id": 7, "snake_start_loc": 21, "prey_start_loc": 0}, {"game_id": 8, "snake_start_loc": 1, "prey_start_loc": 18}, {"game_id": 9, "snake_start_loc": 12, "prey_start_loc": 22}], "dim": 5, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) The snake can move up, down, left, or right exactly one grid space per step. I will give you a grid, and you will tell me every move needed to reach the food in one go.\n4) The snake will be represented using the character \"s\", and the food with character \"*\".\n5) You will give ALL necessary moves in one go, and each individual move MUST use the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via moves in that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": ""}, {"name": "7x7_withplanning", "game_instances": [{"game_id": 0, "snake_start_loc": 46, "prey_start_loc": 43}, {"game_id": 1, "snake_start_loc": 28, "prey_start_loc": 26}, {"game_id": 2, "snake_start_loc": 6, "prey_start_loc": 15}, {"game_id": 3, "snake_start_loc": 17, "prey_start_loc": 7}, {"game_id": 4, "snake_start_loc": 38, "prey_start_loc": 48}, {"game_id": 5, "snake_start_loc": 48, "prey_start_loc": 24}, {"game_id": 6, "snake_start_loc": 3, "prey_start_loc": 35}, {"game_id": 7, "snake_start_loc": 24, "prey_start_loc": 25}, {"game_id": 8, "snake_start_loc": 38, "prey_start_loc": 3}, {"game_id": 9, "snake_start_loc": 44, "prey_start_loc": 48}], "dim": 7, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) The snake can move up, down, left, or right exactly one grid space per step. I will give you a grid, and you will tell me every move needed to reach the food in one go.\n4) The snake will be represented using the character \"s\", and the food with character \"*\".\n5) You will give ALL necessary moves in one go, and each individual move MUST use the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via moves in that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": ""}, {"name": "9x9_withplanning", "game_instances": [{"game_id": 0, "snake_start_loc": 68, "prey_start_loc": 1}, {"game_id": 1, "snake_start_loc": 66, "prey_start_loc": 24}, {"game_id": 2, "snake_start_loc": 47, "prey_start_loc": 11}, {"game_id": 3, "snake_start_loc": 79, "prey_start_loc": 31}, {"game_id": 4, "snake_start_loc": 8, "prey_start_loc": 6}, {"game_id": 5, "snake_start_loc": 1, "prey_start_loc": 73}, {"game_id": 6, "snake_start_loc": 52, "prey_start_loc": 22}, {"game_id": 7, "snake_start_loc": 14, "prey_start_loc": 45}, {"game_id": 8, "snake_start_loc": 19, "prey_start_loc": 32}, {"game_id": 9, "snake_start_loc": 36, "prey_start_loc": 51}], "dim": 9, "max_turns": 20, "navigator_initial_prompt": "We are going to play a simplified game of snake.\n\nHere are the rules:\n1) The snake has no tail, and the snake cannot go outside the defined grid space.\n2) A grid will be provided with a starting position for the snake, and a position representing the \"food\". Your job is to tell me at each step what direction the snake will travel, in single-grid moves.\n3) The snake can move up, down, left, or right exactly one grid space per step. I will give you a grid, and you will tell me every move needed to reach the food in one go.\n4) The snake will be represented using the character \"s\", and the food with character \"*\".\n5) You will give ALL necessary moves in one go, and each individual move MUST use the following format: \nUnder NO circumstances are other forms of dialogue permitted, and your only form of communication is via moves in that format.\n\nAre you ready? Let's begin.", "navigator_reprompt": "You must respond using the format: \n", "navigator_response_pattern": ""}]} \ No newline at end of file diff --git a/games/simplesnake/instancegenerator.py b/games/simplesnake/instancegenerator.py new file mode 100644 index 0000000000..378207d559 --- /dev/null +++ b/games/simplesnake/instancegenerator.py @@ -0,0 +1,99 @@ +import os +import random +import logging +from clemcore.clemgame import GameInstanceGenerator + +MAX_TURNS = 20 +N_INSTANCES = 10 +GAME_NAME = 'simplesnake' +SEED = -413 + +logger = logging.getLogger(__name__) + +LANGUAGE = "en" + + +class SimpleSnakeInstanceGenerator(GameInstanceGenerator): + def __init__(self): + super().__init__(os.path.dirname(__file__)) + + def on_generate(self, variant=None): + matrices = self.load_file('resources/matrices.txt').strip('\n').split('\n') + + for matrix in matrices: + # create new experiment + experiment_name = matrix + if variant is not None: + experiment_name = f'{experiment_name}_with{variant}' + experiment = self.add_experiment(experiment_name) + + # store important game values + dim = int(matrix[0]) + experiment['dim'] = dim + experiment['max_turns'] = MAX_TURNS + + # load needed template resources + prompt_path = 'resources/initial_prompts/navigator_prompt' + if variant is not None: + prompt_path = f'{prompt_path}_with_{variant}' + experiment['navigator_initial_prompt'] = self.load_template(prompt_path) + + prompt_path = 'resources/reprompts/navigator_reprompt' + experiment['navigator_reprompt'] = self.load_template(prompt_path) + + prompt_path = 'resources/patterns/navigator_response' + if variant == 'planning': + prompt_path = f'{prompt_path}_with_{variant}' + experiment['navigator_response_pattern'] = self.load_template(prompt_path) + + instances = [] + for game_id in range(N_INSTANCES): + instance = self.add_game_instance(experiment, game_id) + + snake_start_loc = random.choice(range(dim * dim)) + instance['snake_start_loc'] = snake_start_loc + + possible_prey_locs = list(range(dim * dim)) + possible_prey_locs.remove(snake_start_loc) + prey_start_loc = random.choice(possible_prey_locs) + instance['prey_start_loc'] = prey_start_loc + + # add obstacle information + if variant == 'obstacles': + possible_obs_locs = list(range(dim * dim)) + possible_obs_locs = possible_obs_locs[dim+1:-dim] + + # obs cannot exist on outsider border of grid + possible_obs_locs = [loc for loc in possible_obs_locs if loc % dim != 0 and (loc+1) % dim != 0] + + # obs cannot exist where other entities are + if snake_start_loc in possible_obs_locs: + possible_obs_locs.remove(snake_start_loc) + if prey_start_loc in possible_obs_locs: + possible_obs_locs.remove(prey_start_loc) + + # pick up to two obstacles + obstacle_locs = [] + for _ in range(3): # 3 is the max number of obstacles that doesn't risk being unplayable + if len(possible_obs_locs) > 0: + obstacle_locs.append(random.choice(possible_obs_locs)) + possible_obs_locs.remove(obstacle_locs[-1]) + else: + break + + instance['obstacle_locs'] = obstacle_locs + + instances.append(instance) + + experiment['game_instances'] = instances + + def generate(self, variant, filename): + self.on_generate(variant=variant) + self.store_file(self.instances, filename, sub_dir="in") + + +if __name__ == '__main__': + random.seed(SEED) + SimpleSnakeInstanceGenerator().generate(None, 'instances.json') + SimpleSnakeInstanceGenerator().generate('obstacles', 'instances_withobstacles.json') + SimpleSnakeInstanceGenerator().generate('planning', 'instances_withplanning.json') diff --git a/games/simplesnake/master.py b/games/simplesnake/master.py new file mode 100644 index 0000000000..21967504c8 --- /dev/null +++ b/games/simplesnake/master.py @@ -0,0 +1,334 @@ +import re +import numpy as np +from typing import Dict, List +from clemcore.clemgame import GameSpec, GameMaster, GameBenchmark, Player +from clemcore.clemgame.legacy.master import DialogueGameMaster +from clemcore.clemgame.legacy.scorer import GameScorer +from clemcore.backends.model_registry import Model, CustomResponseModel +from clemcore.clemgame.metrics import METRIC_REQUEST_COUNT_VIOLATED, METRIC_REQUEST_COUNT_PARSED, METRIC_REQUEST_COUNT, \ + METRIC_REQUEST_SUCCESS_RATIO, METRIC_ABORTED, BENCH_SCORE, METRIC_SUCCESS, METRIC_LOSE +import logging +logger = logging.getLogger(__name__) + + +class SimpleSnakeGameBenchmark(GameBenchmark): + def __init__(self, game_spec: GameSpec): + super().__init__(game_spec) + + def get_description(self): + return "Plays a simple game of snake with one player." + + def is_single_player(self) -> bool: + return True + + def create_game_master(self, experiment: Dict, player_models: List[Model]) -> GameMaster: + return SimpleSnake(self.game_spec, experiment, player_models) + + def create_game_scorer(self, experiment: Dict, game_instance: Dict) -> GameScorer: + return SimpleSnakeGameScorer(self.game_name, experiment, game_instance) + + +class Gameboard: + def __init__(self, dim, snake_start_loc, prey_start_loc, obstacles=None): + self.gameboard = [['' for _ in range(dim)] for _ in range(dim)] + self.dim = dim + + # set snake start location + self.snake_pos = [snake_start_loc // dim, snake_start_loc % dim] + self.gameboard[self.snake_pos[0]][self.snake_pos[1]] = 's' + + # set prey start location + self.prey_pos = [prey_start_loc // dim, prey_start_loc % dim] + self.gameboard[prey_start_loc // dim][prey_start_loc % dim] = '*' + + # set obstacle locations + self.obstacles = [] + if obstacles is not None: + for ob in obstacles: + self.obstacles.append([ob // dim, ob % dim]) + self.gameboard[ob // dim][ ob % dim] = 'X' + + def update_gameboard(self, new_loc) -> str: + """Updates the snake location on the gameboard. Returns True if + update results in valid game state, otherwise False. + """ + row, col = new_loc + # check game state + if row == self.prey_pos[0] and col == self.prey_pos[1]: + return 'win state' + if (row < 0 or row >= self.dim) or (col < 0 or col >= self.dim): + return 'lose state' + for ob in self.obstacles: + if row == ob[0] and col == ob[1]: + return 'lose state' + + # move doesn't end game, so update snake pos + self.gameboard[self.snake_pos[0]][self.snake_pos[1]] = '' + self.snake_pos[0], self.snake_pos[1] = row, col + self.gameboard[self.snake_pos[0]][self.snake_pos[1]] = 's' + return self.__str__() + + def __str__(self): + result = '' + for i in range(self.dim): + for j in range(self.dim): + result += f'[{self.gameboard[i][j]}]' + + if i != self.dim - 1: + result += '\n' + return result + + +class Navigator(Player): + def __init__(self, model: Model): + super().__init__(model) + + def _custom_response(self): + return "Alright, I'm ready!" + + +class Describer(Player): + def __init__(self, dim, snake_start_loc, prey_start_loc, obstacle_locs, match_pattern): + super().__init__(CustomResponseModel()) + self.gameboard = Gameboard(dim, snake_start_loc, prey_start_loc, obstacle_locs) + self.init_game_state = True + self.pattern = match_pattern + + def _custom_response(self, context: Dict) -> str: + if self.init_game_state: + self.init_game_state = False + return self.gameboard.__str__() + + # parse direction and determine new snake location + directions = re.findall(self.pattern, context['content'].lower()) + row, col = 0, 0 + state = None + for direction in directions: + if direction == 'up': + row = self.gameboard.snake_pos[0] - 1 + col = self.gameboard.snake_pos[1] + elif direction == 'down': + row = self.gameboard.snake_pos[0] + 1 + col = self.gameboard.snake_pos[1] + elif direction == 'left': + row = self.gameboard.snake_pos[0] + col = self.gameboard.snake_pos[1] - 1 + elif direction == 'right': + row = self.gameboard.snake_pos[0] + col = self.gameboard.snake_pos[1] + 1 + else: + raise RuntimeError('Failed to parse direction.') + state = self.gameboard.update_gameboard((row, col)) + + # return self.gameboard.update_gameboard((row, col)) + return state + + +class SimpleSnake(DialogueGameMaster): + """ Implements a game of snake in which one player describes the directions in which + to move a snake within a grid to find its food. + """ + + def __init__(self, game_spec: GameSpec, experiment: Dict, player_models: List[Model] = None): + super().__init__(game_spec, experiment, player_models) + # experiment-level variables + self.max_turns = experiment['max_turns'] + self.dim = experiment['dim'] + self.navigator_initial_prompt = experiment['navigator_initial_prompt'] + self.navigator_response_pattern = experiment['navigator_response_pattern'] + self.navigator_reprompt = experiment['navigator_reprompt'] + + def _on_setup(self, **game_instance): + self.pass_turn = True + self.is_warned = False + + # instance-level flags + self.invalid_response = False + + # instance-level variables + self.game_instance = game_instance + self.snake_location = game_instance['snake_start_loc'] + self.prey_location = game_instance['prey_start_loc'] + + + # instance players + self.navigator = Navigator(self.player_models[0]) + self.add_player(self.navigator) + + # if 'obstacles' in self.game_name: + if self.game_spec.game_name == 'simplesnake_withobstacles': + self.obstacle_locs = game_instance['obstacle_locs'] + else: + self.obstacle_locs = None + + self.describer = Describer( + dim=self.dim, + snake_start_loc=self.snake_location, + prey_start_loc=self.prey_location, + obstacle_locs=self.obstacle_locs, + match_pattern=self.navigator_response_pattern, + ) + self.add_player(self.describer) + + def _on_before_game(self): + self.set_context_for(self.navigator, self.navigator_initial_prompt) + + def _does_game_proceed(self): + """Proceed as long as the snake does not occupy the same gridspace as the prey.""" + if self.invalid_response: + self.log_to_self("invalid_response", "abort game") + return False + if self.win_state: + self.log_to_self("game_win", "end game") + return False + if self.lose_state: + self.log_to_self("game_loss", "end game") + return False + if self.current_round >= self.max_turns: + self.log_to_self("max_turns_reached", str(self.max_turns)) + return False + return True + + def _validate_player_response(self, player: Player, utterance: str) -> bool: + # reset flags + self.invalid_response = False + self.win_state = False + self.lose_state = False + bad_response = False + + if player == self.navigator: + if self.current_round == 0: # ensure grid is sent during initial round + self.log_to_self("game start", "Get initial grid.") + return True + + # is navigator response in valid format? + if self.game_spec.game_name == 'simplesnake_withplanning': + bad_response = re.fullmatch(f'^(?:{self.navigator_response_pattern}\s*)+$', utterance.lower()) is None + else: + matches = re.findall(self.navigator_response_pattern, utterance.lower()) + bad_response = len(matches) == 0 or len(matches) > 1 + + if bad_response: + if self.is_warned: # Reprompting failed + self.invalid_response = True + return False + else: # Skip describer's turn and reprompt navigator + self.pass_turn = False + self.set_context_for(self.navigator, self.navigator_reprompt) + self.is_warned = True + return True + + # Reset flags after they've been used to skip describer + self.is_warned = False + self.pass_turn = True + elif player == self.describer: + # handle end-of-game describer response + if self.game_spec.game_name == 'simplesnake_withplanning': + if self.current_round == 1: + if utterance == 'win state': + self.win_state = True + else: + self.lose_state = True + return False + else: + if utterance == 'lose state': + self.lose_state = True + return False + if utterance == 'win state': + self.win_state = True + return False + + return True # valid response + + def _on_valid_player_response(self, player: Player, parsed_response: str): + if player == self.navigator: + if self.current_round == 0: + self.set_context_for(self.describer, 'Get initial grid.') + else: + self.set_context_for(self.describer, parsed_response) + elif player == self.describer: + self.set_context_for(self.navigator, parsed_response) + + def _should_pass_turn(self): + return self.pass_turn + + +class SimpleSnakeGameScorer(GameScorer): + """GameScorer subclass for SimpleSnake. + Reads episode records, counts failures, calculates scores and stores the + results in score files.""" + + def __init__(self, game_name: str, experiment: Dict, game_instance: Dict): + super().__init__(game_name, experiment, game_instance) + self.max_turns = experiment['max_turns'] + + def compute_scores(self, episode_interactions: Dict) -> None: + """Episode level scores. + Writes to score file in the episode directory. + """ + bench_score = None + turn_scores = [] + invalid_response = False + win_state = False + lose_state = False + max_turns_reached = False + + for turn_idx, turn in enumerate(episode_interactions['turns']): + turn_score = {'request_count': 1} + + for event in turn: + action = event['action'] + if action['type'] == 'invalid_response': + invalid_response = True + if action['type'] == 'game_loss': + lose_state = True + if action['type'] == 'game_win': + win_state = True + if action['type'] == 'max_turns_reached': + max_turns_reached = True + + # check if LM followed correct formatting rules + if invalid_response: + turn_score['violated_request_count'] = 1 + turn_score['parsed_request_count'] = 0 + else: + turn_score['violated_request_count'] = 0 + turn_score['parsed_request_count'] = 1 + + self.log_turn_score(turn_idx, METRIC_REQUEST_COUNT, turn_score["request_count"]) + self.log_turn_score(turn_idx, METRIC_REQUEST_COUNT_PARSED, turn_score["parsed_request_count"]) + self.log_turn_score(turn_idx, METRIC_REQUEST_COUNT_VIOLATED, turn_score["violated_request_count"]) + turn_scores.append(turn_score) + + violated_request_count = sum([turn["violated_request_count"] for turn in turn_scores]) + self.log_episode_score(METRIC_REQUEST_COUNT_VIOLATED, violated_request_count) + + parsed_request_count = sum([turn["parsed_request_count"] for turn in turn_scores]) + self.log_episode_score(METRIC_REQUEST_COUNT_PARSED, parsed_request_count) + + request_count = sum([turn["request_count"] for turn in turn_scores]) + self.log_episode_score(METRIC_REQUEST_COUNT, request_count) + + self.log_episode_score(METRIC_REQUEST_SUCCESS_RATIO, parsed_request_count / request_count) + + # Common metrics + if invalid_response: # whether a violation of the game rules happened (response not parsable) + self.log_episode_score(METRIC_ABORTED, 1) + self.log_episode_score(METRIC_SUCCESS, 0) + self.log_episode_score(METRIC_LOSE, 0) + bench_score = np.nan # metric not applicable + else: + if win_state: # no rule violations, and reached goal + self.log_episode_score(METRIC_SUCCESS, 1) + self.log_episode_score(METRIC_LOSE, 0) + bench_score = 100 # assign 100% score for game win + elif lose_state: # game was lost + self.log_episode_score(METRIC_SUCCESS, 0) + self.log_episode_score(METRIC_LOSE, 1) + bench_score = 0 + elif max_turns_reached: # goal was never found and turns were exceeded + self.log_episode_score(METRIC_SUCCESS, 0) + self.log_episode_score(METRIC_LOSE, 1) + bench_score = 0 + self.log_episode_score(METRIC_ABORTED, 0) + self.log_episode_score(BENCH_SCORE, bench_score) diff --git a/games/simplesnake/resources/initial_prompts/navigator_prompt.template b/games/simplesnake/resources/initial_prompts/navigator_prompt.template new file mode 100644 index 0000000000..9b166c933b --- /dev/null +++ b/games/simplesnake/resources/initial_prompts/navigator_prompt.template @@ -0,0 +1,11 @@ +We are going to play a simplified game of snake. + +Here are the rules: +1) The snake has no tail, and the snake cannot go outside the defined grid space. +2) A grid will be provided with a starting position for the snake, and a position representing the "food". Your job is to tell me at each step what direction the snake will travel, in single-grid moves. +3) The snake can move up, down, left, or right exactly one grid space per step. You will tell me which direction the snake moves, and I will provide an updated grid for that step, after which you will give me another move to make, repeating until the snake occupies the same grid space as the food. +4) The snake will be represented using the character "s", and the food with character "*". +5) You will only give one response at a time, and you will ONLY respond using the following format: +Under NO circumstances are other forms of dialogue permitted, and your only form of communication is via that format. + +Are you ready? Let's begin. \ No newline at end of file diff --git a/games/simplesnake/resources/initial_prompts/navigator_prompt_with_obstacles.template b/games/simplesnake/resources/initial_prompts/navigator_prompt_with_obstacles.template new file mode 100644 index 0000000000..68fc834767 --- /dev/null +++ b/games/simplesnake/resources/initial_prompts/navigator_prompt_with_obstacles.template @@ -0,0 +1,12 @@ +We are going to play a simplified game of snake. + +Here are the rules: +1) The snake has no tail, and the snake cannot go outside the defined grid space. +2) A grid will be provided with a starting position for the snake, and a position representing the "food". Your job is to tell me at each step what direction the snake will travel, in single-grid moves. +3) There are obstacles on the grid that must be avoided. If the snake touches the obstacles, you lose the game. +4) The snake can move up, down, left, or right exactly one grid space per step. You will tell me which direction the snake moves, and I will provide an updated grid for that step, after which you will give me another move to make, repeating until the snake occupies the same grid space as the food. +5) The snake will be represented using the character "s", the food with character "*", and obstacles with "X". +6) You will only give one response at a time, and you will ONLY respond using the following format: +Under NO circumstances are other forms of dialogue permitted, and your only form of communication is via that format. + +Are you ready? Let's begin. \ No newline at end of file diff --git a/games/simplesnake/resources/initial_prompts/navigator_prompt_with_planning.template b/games/simplesnake/resources/initial_prompts/navigator_prompt_with_planning.template new file mode 100644 index 0000000000..8ba819a85c --- /dev/null +++ b/games/simplesnake/resources/initial_prompts/navigator_prompt_with_planning.template @@ -0,0 +1,11 @@ +We are going to play a simplified game of snake. + +Here are the rules: +1) The snake has no tail, and the snake cannot go outside the defined grid space. +2) A grid will be provided with a starting position for the snake, and a position representing the "food". Your job is to tell me at each step what direction the snake will travel, in single-grid moves. +3) The snake can move up, down, left, or right exactly one grid space per step. I will give you a grid, and you will tell me every move needed to reach the food in one go. +4) The snake will be represented using the character "s", and the food with character "*". +5) You will give ALL necessary moves in one go, and each individual move MUST use the following format: +Under NO circumstances are other forms of dialogue permitted, and your only form of communication is via moves in that format. + +Are you ready? Let's begin. \ No newline at end of file diff --git a/games/simplesnake/resources/matrices.txt b/games/simplesnake/resources/matrices.txt new file mode 100644 index 0000000000..9d437dca52 --- /dev/null +++ b/games/simplesnake/resources/matrices.txt @@ -0,0 +1,3 @@ +5x5 +7x7 +9x9 \ No newline at end of file diff --git a/games/simplesnake/resources/patterns/navigator_response.template b/games/simplesnake/resources/patterns/navigator_response.template new file mode 100644 index 0000000000..5866659cb5 --- /dev/null +++ b/games/simplesnake/resources/patterns/navigator_response.template @@ -0,0 +1 @@ +^$ \ No newline at end of file diff --git a/games/simplesnake/resources/patterns/navigator_response_with_planning.template b/games/simplesnake/resources/patterns/navigator_response_with_planning.template new file mode 100644 index 0000000000..eced2a0b17 --- /dev/null +++ b/games/simplesnake/resources/patterns/navigator_response_with_planning.template @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/games/simplesnake/resources/reprompts/navigator_reprompt.template b/games/simplesnake/resources/reprompts/navigator_reprompt.template new file mode 100644 index 0000000000..53743f724a --- /dev/null +++ b/games/simplesnake/resources/reprompts/navigator_reprompt.template @@ -0,0 +1 @@ +You must respond using the format: diff --git a/run_memory_experiments.py b/run_memory_experiments.py new file mode 100644 index 0000000000..63eae04758 --- /dev/null +++ b/run_memory_experiments.py @@ -0,0 +1,41 @@ +import os + +igfile = 'instancegenerator.py' +max_instances = 2 + +games = [ + # 'memory', + # 'memory_narrative', + # 'memory_turns', + 'memory_narrative_turns' +] + +models = [ + 'EuroLLM-1.7B-Instruct', + 'gemini-1.5-flash-001', + 'gemini-1.5-flash-002', + 'deepseek-llm-7b-chat', + 'koala-13B-HF', + 'oasst-sft-4-pythia-12b-epoch-3.5', + 'llama-2-7b-chat-hf', + 'Phi-3-mini-128k-instruct' + # 'Qwen1.5-0.5B-Chat-GGUF-q8' # issues with llama_ccp + +] + +def run_experiments(game, model, n_instances): + exp_results = f'results/{game}_{model}_{n_instances}' + os.system(f'python cli.py run -g {game} -m {model} -r {exp_results}') + os.system(f'python cli.py score -r {exp_results}') + +for game in games: + for n_instances in range(2,max_instances+1): + os.environ['NINSTANCES'] = str(n_instances) + os.system(f'cp games/{game}/{igfile} {igfile}') + os.system(f'python {igfile}') + for model in models: + print(f"running {game} {n_instances} {model}") + run_experiments(game, model, n_instances) + +os.system(f'rm {igfile}') +os.system('python bencheval.py -p results')