From 07fdde70c10388628b40f01e048fb420caff773f Mon Sep 17 00:00:00 2001 From: Aamir ali Khan Date: Sat, 29 Jul 2023 10:22:01 +0200 Subject: [PATCH 01/42] Update: Pydantic version from 1.10.4 to 1.10.7 (#3615) --- backend/requirements.txt | 4 ++-- oasst-data/pyproject.toml | 2 +- oasst-shared/pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index b5cf3ca99..79765cd53 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -10,8 +10,8 @@ loguru==0.6.0 numpy>=1.23.2 prometheus-fastapi-instrumentator==5.9.1 psycopg2-binary==2.9.5 -pydantic==1.10.4 -pydantic[email]==1.10.4 +pydantic==1.10.7 +pydantic[email]==1.10.7 python-dotenv==0.21.0 python-jose[cryptography]==3.3.0 redis==4.5.5 diff --git a/oasst-data/pyproject.toml b/oasst-data/pyproject.toml index 418517648..2f9a1a9d1 100644 --- a/oasst-data/pyproject.toml +++ b/oasst-data/pyproject.toml @@ -6,7 +6,7 @@ authors = [ { name = "LAION-AI", email = "contact@laion.ai" } ] dependencies = [ - "pydantic>=1.10.4", + "pydantic==1.10.7", "loguru==0.6.0", "datasets>=2.12.0" ] diff --git a/oasst-shared/pyproject.toml b/oasst-shared/pyproject.toml index 85f3cee84..baee4f9e1 100644 --- a/oasst-shared/pyproject.toml +++ b/oasst-shared/pyproject.toml @@ -6,7 +6,7 @@ authors = [ { name = "LAION-AI", email = "contact@laion.ai" } ] dependencies = [ - "pydantic==1.10.4", + "pydantic==1.10.7", "aiohttp==3.8.3", "aiohttp[speedups]", "loguru==0.6.0", From c2c1318694e35566473c69d4a9cc374a86cff12c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sat, 29 Jul 2023 11:16:00 +0200 Subject: [PATCH 02/42] Changes for orcacode experiment (#3612) --- model/model_training/configs/config.yaml | 72 +++++++++++++++++++ .../custom_datasets/__init__.py | 5 +- .../custom_datasets/instruction.py | 7 +- .../custom_datasets/prompt_dialogue.py | 68 +++++++++--------- .../custom_datasets/qa_datasets.py | 34 ++++++++- model/model_training/models/patching.py | 5 -- model/model_training/trainer_sft.py | 3 + 7 files changed, 150 insertions(+), 44 deletions(-) diff --git a/model/model_training/configs/config.yaml b/model/model_training/configs/config.yaml index a7c1f70c5..6b94a81c3 100644 --- a/model/model_training/configs/config.yaml +++ b/model/model_training/configs/config.yaml @@ -847,3 +847,75 @@ falcon_7b_ntk_test: alpha: 2 datasets: - dolly15k + +llama2_13b_orcacode2_8k: + rng_seed: 0xe1291f21 + random_offset_probability: 0.0 + use_custom_sampler: true + sort_by_length: false + dtype: fp16 + log_dir: "llama2_log_13b_orcacode2_8k" + output_dir: llama2_13b_orcacode2_8k + learning_rate: 1e-5 + model_name: OpenAssistant/llama2-13b-orca-8k-3319 + deepspeed_config: configs/zero_config_pretrain.json + weight_decay: 1e-6 + max_length: 8192 + warmup_steps: 100 + peft_model: false + use_flash_attention: true + gradient_checkpointing: true + gradient_accumulation_steps: 4 + per_device_train_batch_size: 2 + per_device_eval_batch_size: 1 + residual_dropout: 0.0 + eval_steps: 200 + save_steps: 500 # (total steps: 1558, bs: 64) + num_train_epochs: 1 + save_total_limit: 4 + datasets: + - dolphin-mix: + num_samples: 1000000 # total entries 2840090 + max_char_len: 32000 + val_split: 0.1 + max_val_set: 2000 + seed: 44 + - oasst_export: + lang: "bg,ca,cs,da,de,en,es,fr,hr,hu,it,nl,pl,pt,ro,ru,sl,sr,sv,uk" + input_file_path: 2023-07-23_oasst_ready.tar.gz + top_k: 1 + val_split: 0.05 + - wizard_evol_instruct_v2: + val_split: 0.01 + fraction: 0.1 + - evol-codealpaca-v1: + fill_min_length: 20000 + val_split: 0.1 + - cot_submix_original: + fill_min_length: 20000 + val_split: 0.1 + - megacode: + fill_min_length: 24000 + val_split: 0.1 + max_val_set: 1000 + - evol_instruct_code: + fill_min_length: 24000 + val_split: 0.1 + max_val_set: 1000 + # Dataset composition: + # Train: + # dolphin-mix: 40374 + # oasst_export: 11441 + # wizard_evol_instruct_v2: 15236 + # evol-codealpaca-v1: 5623 + # cot_submix_original: 8651 + # megacode: 14320 + # evol_instruct_code: 4093 + # Valid: + # dolphin-mix: 2000 + # oasst_export: 603 + # wizard_evol_instruct_v2: 1540 + # evol-codealpaca-v1: 625 + # cot_submix_original: 962 + # megacode: 1000 + # evol_instruct_code: 455 diff --git a/model/model_training/custom_datasets/__init__.py b/model/model_training/custom_datasets/__init__.py index 3043a5db1..29eb24f3b 100644 --- a/model/model_training/custom_datasets/__init__.py +++ b/model/model_training/custom_datasets/__init__.py @@ -20,6 +20,7 @@ TranslatedQA, Vicuna, WebGPT, + WizardEvolInstructV2, load_alpaca_dataset, ) from model_training.custom_datasets.rank_datasets import AugmentedOA @@ -110,7 +111,7 @@ def get_one_dataset( eval = SummarizationDataset(dataset_name, data_path, "validation") train = dataset elif dataset_name in INSTRUCTION_DATASETS: - dataset = InstructionDataset(dataset_name, data_path, "train") + dataset = InstructionDataset(dataset_name, data_path, "train", **kwargs) elif "ted_trans" in dataset_name: language_pair = dataset_name.split("_")[-1] dataset = TEDTalk(pair=language_pair, split="train") @@ -143,6 +144,8 @@ def get_one_dataset( dataset = TranslatedQA(data_path) elif dataset_name == "vicuna": dataset = Vicuna(cache_dir=data_path, **kwargs) + elif dataset_name == "wizard_evol_instruct_v2": + dataset = WizardEvolInstructV2(cache_dir=data_path, **kwargs) elif dataset_name == "oasst_export": train, eval = load_oasst_export(data_path=data_path, val_split=val_split, mode=mode, **kwargs) elif dataset_name == "hf_summary": diff --git a/model/model_training/custom_datasets/instruction.py b/model/model_training/custom_datasets/instruction.py index c6b6acd8d..7b6ad3978 100644 --- a/model/model_training/custom_datasets/instruction.py +++ b/model/model_training/custom_datasets/instruction.py @@ -30,6 +30,8 @@ "wizardlm_70k": "ehartford/WizardLM_alpaca_evol_instruct_70k_unfiltered", "megacode": "rombodawg/MegaCodeTraining112k", "evol_instruct_code": "nickrosh/Evol-Instruct-Code-80k-v1", + "evol-codealpaca-v1": "theblackcat102/evol-codealpaca-v1", + "cot_submix_original": "conceptofmind/cot_submix_original", } @@ -42,9 +44,12 @@ def __init__(self, dataset, cache_dir, split, mode="sft", fill_min_length: Optio if dataset == "minimath": self.instruction_column = "question" self.response_column = "answer" - elif dataset in ("wizardlm_70k", "evol_instruct_code"): + elif dataset in ("wizardlm_70k", "evol_instruct_code", "evol-codealpaca-v1"): self.instruction_column = "instruction" self.response_column = "output" + elif dataset == "cot_submix_original": + self.instruction_column = "inputs" + self.response_column = "targets" elif dataset == "megacode": self.instruction_column = "prompt" self.response_column = "completion" diff --git a/model/model_training/custom_datasets/prompt_dialogue.py b/model/model_training/custom_datasets/prompt_dialogue.py index 476e82b97..1d30458cb 100644 --- a/model/model_training/custom_datasets/prompt_dialogue.py +++ b/model/model_training/custom_datasets/prompt_dialogue.py @@ -2,9 +2,8 @@ import json import re from pathlib import Path -from typing import List, Optional, Union +from typing import List, Mapping, Optional, Sequence, Union -import numpy as np import requests from datasets import load_dataset from model_training.custom_datasets.formatting import DatasetEntrySft, Role, Utterance @@ -199,45 +198,46 @@ def __getitem__(self, idx): class DolphinMix(Dataset): name = "dophin-mix" - def __init__(self, cache_dir, num_samples=100000, max_char_len=8000, seed=42): - self.dataset = load_dataset( - "ehartford/dolphin", data_files="flan5m-alpaca-uncensored.jsonl", cache_dir=cache_dir - ) - self.dataset = self.dataset["train"].shuffle(seed).select(range(num_samples)) + def __init__( + self, + cache_dir: Optional[str] = None, + num_samples: Optional[int] = None, + max_char_len: int = 8000, + seed: int = 42, + data_files: Union[ + str, Sequence[str], Mapping[str, Union[str, Sequence[str]]] + ] = "flan5m-alpaca-uncensored.jsonl", + split: str = "train", + ): + # flan5m-alpaca-uncensored.jsonl has total entries 2840090 + self.dataset = load_dataset("ehartford/dolphin", data_files=data_files, cache_dir=cache_dir) + self.dataset = self.dataset[split].shuffle(seed).flatten_indices() + if num_samples: + self.dataset = self.dataset.select(range(num_samples)) self.max_char_len = max_char_len - instructions = set([item["instruction"] for item in self.dataset]) + instructions = sorted(set([item["instruction"] for item in self.dataset])) self.conversations = [] for inst in instructions: data_sample = self.dataset.filter(lambda example: example["instruction"] == inst) - available_indices = np.arange(0, len(data_sample)).tolist() - removed_indices = [] - for idx in available_indices: - conversation_len = len(inst) - if idx not in removed_indices and conversation_len < self.max_char_len: - conversation = {"conversation": []} - conversation["instruction"] = inst - input, output = [data_sample[idx][key] for key in ("input", "output")] - conversation["conversation"].append({"input": input, "output": output}) - conversation_len += len(input) + len(output) - removed_indices.append(idx) - while conversation_len < self.max_char_len: - indices_to_pick = np.setdiff1d(available_indices, removed_indices) - if len(indices_to_pick) > 0: - idx = np.random.choice(indices_to_pick, size=1)[0] - input, output = [data_sample[int(idx)][key] for key in ("input", "output")] - conversation["conversation"].append({"input": input, "output": output}) - conversation_len += len(input) + len(output) - removed_indices.append(idx) - else: - break - - self.conversations.append(conversation) - - def __len__(self): + conversation_len = len(inst) + conversation = [] + for entry in data_sample: + input, output = entry["input"], entry["output"] + conversation.append({"input": input, "output": output}) + conversation_len += len(input) + len(output) + if conversation_len >= self.max_char_len: + self.conversations.append({"conversation": conversation, "instruction": inst}) + conversation_len = len(inst) + conversation = [] + + if len(conversation) > 0: + self.conversations.append({"conversation": conversation, "instruction": inst}) + + def __len__(self) -> int: return len(self.conversations) - def __getitem__(self, idx): + def __getitem__(self, idx) -> DatasetEntrySft: conversation, instruction = [self.conversations[idx][key] for key in ("conversation", "instruction")] conversation = [(item["input"], item["output"]) for item in conversation] conversation = list(sum(conversation, ())) diff --git a/model/model_training/custom_datasets/qa_datasets.py b/model/model_training/custom_datasets/qa_datasets.py index 3d7a1f066..8de847ea8 100644 --- a/model/model_training/custom_datasets/qa_datasets.py +++ b/model/model_training/custom_datasets/qa_datasets.py @@ -514,9 +514,8 @@ def process_vicuna_conversations( def __init__(self, cache_dir: str | Path, mode: str = "sft", input_max_length: int = 32 * 1024) -> None: super().__init__() - self.pairs = [] - if mode not in ("sft", "rl"): - raise NotImplementedError(f"Currently only the modes 'sft' and 'rl' are implemented. Received {mode}.") + if mode != "sft": + raise NotImplementedError(f"Currently only the mode 'sft' is implemented. Received {mode}.") self.mode = mode dataset = load_dataset( @@ -526,8 +525,37 @@ def __init__(self, cache_dir: str | Path, mode: str = "sft", input_max_length: i revision="7b8551404f3de5704d634e7516b9ff77be3e2700", )["train"] + self.pairs = [] for data in dataset: if (qa := self.process_vicuna_conversations(data, input_max_length=input_max_length)) is not None: + if len(qa[0]) > 0 and len(qa[0]) == len(qa[1]): + self.pairs.append(create_dataset_entry_qa(mode=self.mode, questions=qa[0], answers=qa[1])) + + def __len__(self) -> int: + return len(self.pairs) + + def __getitem__(self, index: int) -> DatasetEntry: + return self.pairs[index] + + +class WizardEvolInstructV2(Dataset): + def __init__(self, cache_dir: str | Path, mode: str = "sft", input_max_length: int = 32 * 1024) -> None: + super().__init__() + + if mode != "sft": + raise NotImplementedError(f"Currently only the mode 'sft' is implemented. Received {mode}.") + self.mode = mode + + dataset = load_dataset( + "ehartford/WizardLM_evol_instruct_V2_196k_unfiltered_merged_split", + cache_dir=cache_dir, + data_files=["WizardLM_evol_instruct_V2_196k_unfiltered_merged_split.json"], + revision="34f04cfbc280da93a79ad9ecf339923f9411c1fc", + )["train"] + + self.pairs = [] + for data in dataset: + if (qa := Vicuna.process_vicuna_conversations(data, input_max_length=input_max_length)) is not None: if len(qa[0]) > 0 and len(qa[0]) == len(qa[1]): self.pairs.append(create_dataset_entry_qa(mode="sft", questions=qa[0], answers=qa[1], lang="en")) diff --git a/model/model_training/models/patching.py b/model/model_training/models/patching.py index bda61501a..cdb8b7598 100644 --- a/model/model_training/models/patching.py +++ b/model/model_training/models/patching.py @@ -236,9 +236,6 @@ def from_config(cls, config): args = config.superhot_config return cls(model_name, **args) - def update_config(self, model, scaling_factor): - model.config["rope_scaling"] = {"type": self.rope_type, "factor": scaling_factor} - def patch(self, model): if self.architecture == "FalconForCausalLM": self.patch_falcon_model(model, **self.args) @@ -247,8 +244,6 @@ def patch(self, model): else: raise NotImplementedError() - self.update_config(model, self.args.get("scaling_factor")) - def patch_falcon_model(self, model, **kwargs): for each in model.transformer.h: each.self_attention.maybe_rotary = self.patch_fun(model.config.head_dim, **kwargs) diff --git a/model/model_training/trainer_sft.py b/model/model_training/trainer_sft.py index 4c4c82099..0e0bef800 100755 --- a/model/model_training/trainer_sft.py +++ b/model/model_training/trainer_sft.py @@ -422,6 +422,9 @@ def main(): if superhot: superhot.patch(model) + print(f"rope_scaling: {model.config.rope_scaling}") + print(f"max_position_embeddings: {model.config.max_position_embeddings}") + if training_conf.peft_model: print("Using PEFT model") model = peft_model(model, training_conf) From b9ac30a89fea29b3e493eeb24e6611413b293b90 Mon Sep 17 00:00:00 2001 From: Shahul ES Date: Sat, 29 Jul 2023 23:00:03 +0530 Subject: [PATCH 03/42] Added support for LLongMA (#3616) Added tokenizer for llongma --- model/model_training/utils/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model/model_training/utils/utils.py b/model/model_training/utils/utils.py index e3c909889..677da36a2 100644 --- a/model/model_training/utils/utils.py +++ b/model/model_training/utils/utils.py @@ -187,6 +187,7 @@ class TokenizerConfig(NamedTuple): "falcon": TokenizerConfig( special_tokens=SpecialTokens("<|endoftext|>", "<|endoftext|>", sep_token="<|endoftext|>") ), + "LLongMA": TokenizerConfig(special_tokens=SpecialTokens("", "", sep_token="")), } From 7a68b591a12fa900ac58601ff5b77519c97aad97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dragan=20Jovanovi=C4=87?= Date: Sun, 30 Jul 2023 23:17:54 +0200 Subject: [PATCH 04/42] Custom instructions feature (#3597) Added basic functionality of "custom-instructions" It utilises two additional fields in work_parameters: - user_profile (here user would describe what he wants to share with llm, for chat session) - user_response_instructions (here user would describe how they want llm to respond to questions and queries, like format, style, length, etc...) Here are some of the current UI changes introduced with this PR.
Custom instructions Screenshot 2023-07-23 at 21 46 36 Screenshot 2023-07-23 at 21 46 42 Screenshot 2023-07-23 at 21 47 16 Screenshot 2023-07-23 at 21 48 21 Screenshot 2023-07-23 at 21 48 29
--------- Co-authored-by: Oliver Stanley --- .../oasst_inference_server/routes/chats.py | 2 + .../oasst_inference_server/schemas/chat.py | 2 + inference/worker/chat_chain.py | 43 ++++++- inference/worker/chat_chain_prompts.py | 10 ++ inference/worker/chat_chain_utils.py | 5 +- inference/worker/work.py | 13 ++- .../oasst_shared/schemas/inference.py | 2 + website/public/locales/en/chat.json | 7 +- .../src/components/Chat/ChatConfigForm.tsx | 42 ++++++- .../src/components/Chat/ChatConfigSaver.tsx | 16 ++- .../src/components/Chat/ChatConversation.tsx | 8 +- .../components/Chat/CustomInstructions.tsx | 106 ++++++++++++++++++ website/src/types/Chat.ts | 10 ++ website/src/utils/chat.ts | 3 +- 14 files changed, 253 insertions(+), 16 deletions(-) create mode 100644 website/src/components/Chat/CustomInstructions.tsx diff --git a/inference/server/oasst_inference_server/routes/chats.py b/inference/server/oasst_inference_server/routes/chats.py index 8b151074b..1959b094e 100644 --- a/inference/server/oasst_inference_server/routes/chats.py +++ b/inference/server/oasst_inference_server/routes/chats.py @@ -153,6 +153,8 @@ async def create_assistant_message( system_prompt=request.system_prompt, plugins=request.plugins, plugin_max_depth=settings.plugin_max_depth, + user_profile=request.user_profile, + user_response_instructions=request.user_response_instructions, ) assistant_message = await ucr.initiate_assistant_message( parent_id=request.parent_id, diff --git a/inference/server/oasst_inference_server/schemas/chat.py b/inference/server/oasst_inference_server/schemas/chat.py index 74ec4a09a..c8a266ef7 100644 --- a/inference/server/oasst_inference_server/schemas/chat.py +++ b/inference/server/oasst_inference_server/schemas/chat.py @@ -15,6 +15,8 @@ class CreateAssistantMessageRequest(pydantic.BaseModel): model_config_name: str sampling_parameters: inference.SamplingParameters = pydantic.Field(default_factory=inference.SamplingParameters) system_prompt: str | None = None + user_profile: str | None = None + user_response_instructions: str | None = None plugins: list[inference.PluginEntry] = pydantic.Field(default_factory=list[inference.PluginEntry]) used_plugin: inference.PluginUsed | None = None diff --git a/inference/worker/chat_chain.py b/inference/worker/chat_chain.py index c4a540697..70c732bbb 100644 --- a/inference/worker/chat_chain.py +++ b/inference/worker/chat_chain.py @@ -6,6 +6,7 @@ import websocket from chat_chain_prompts import ( ASSISTANT_PREFIX, + CUSTOM_INSTRUCTIONS_PREFIX, HUMAN_PREFIX, JSON_FORMAT_NO_PAYLOAD, JSON_FORMAT_PAYLOAD, @@ -56,6 +57,7 @@ def __init__( tool_names: list[str], language: str, action_input_format: str, + custom_instructions: str = "", ): self.tokenizer = tokenizer self.worker_config = worker_config @@ -66,6 +68,7 @@ def __init__( self.language = language self.action_input_format = action_input_format self.current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + self.custom_instructions = custom_instructions def call(self, prompt: str) -> tuple[str, str]: """Prepares and truncates prompt, calls LLM, returns used prompt and response.""" @@ -79,6 +82,7 @@ def call(self, prompt: str) -> tuple[str, str]: self.tokenizer, self.worker_config, self.action_input_format, + self.custom_instructions, ) # We do not strip() outputs as it seems to degrade instruction-following abilities of the model @@ -111,6 +115,7 @@ def handle_plugin_usage( plugin_max_depth: int, ws: websocket.WebSocket, work_request_id: str, + custom_instructions: str = "", ) -> tuple[str, inference.PluginUsed]: execution_details = inference.PluginExecutionDetails( inner_monologue=[], @@ -142,7 +147,15 @@ def handle_plugin_usage( tool_names = [tool.name for tool in tools] chain = PromptedLLM( - tokenizer, worker_config, parameters, prompt_template, memory, tool_names, language, action_input_format + tokenizer, + worker_config, + parameters, + prompt_template, + memory, + tool_names, + language, + action_input_format, + custom_instructions, ) # send "thinking..." intermediate step to UI (This will discard queue position 0) immediately @@ -245,6 +258,7 @@ def handle_plugin_usage( tokenizer, worker_config, action_input_format, + custom_instructions, ) inner_prompt = f"{inner_prompt}\n{THOUGHT_SEQ} I now know the final answer\n{ASSISTANT_PREFIX}: " @@ -296,6 +310,7 @@ def handle_standard_usage( memory: ConversationBufferMemory, worker_config: inference.WorkerConfig, tokenizer: transformers.PreTrainedTokenizer, + custom_instructions: str = "", ): eos_token = tokenizer.eos_token if hasattr(tokenizer, "eos_token") else "" current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -306,7 +321,16 @@ def handle_standard_usage( ) input = f"{original_prompt}{eos_token}{V2_ASST_PREFIX}" init_prompt = prepare_prompt( - input, prompt_template, memory, None, current_time, language, tokenizer, worker_config, action_input_format + input, + prompt_template, + memory, + None, + current_time, + language, + tokenizer, + worker_config, + action_input_format, + custom_instructions, ) return init_prompt, None @@ -355,11 +379,21 @@ def handle_conversation( "language", "current_time", "action_input_format", + "custom_instructions", ] + (["tools_names"] if plugin_enabled else []) # TODO: Consider passing language from the UI here prompt_template = PromptTemplate(input_variables=input_variables, template=TEMPLATE) + custom_instructions = ( + f"""\n{CUSTOM_INSTRUCTIONS_PREFIX.format( + user_profile=work_request.parameters.user_profile, + user_response_instructions=work_request.parameters.user_response_instructions, + )}""" + if work_request.parameters.user_response_instructions or work_request.parameters.user_profile + else "" + ) + if plugin_enabled: return handle_plugin_usage( original_prompt, @@ -374,9 +408,12 @@ def handle_conversation( work_request.parameters.plugin_max_depth, ws, work_request.id, + custom_instructions, ) - return handle_standard_usage(original_prompt, prompt_template, language, memory, worker_config, tokenizer) + return handle_standard_usage( + original_prompt, prompt_template, language, memory, worker_config, tokenizer, custom_instructions + ) except Exception as e: logger.error(f"Error while handling conversation: {e}") return "", None diff --git a/inference/worker/chat_chain_prompts.py b/inference/worker/chat_chain_prompts.py index 299dfa8f5..eccfd7c1e 100644 --- a/inference/worker/chat_chain_prompts.py +++ b/inference/worker/chat_chain_prompts.py @@ -9,6 +9,15 @@ START_SEQ = "Begin!" END_SEQ = "End!" +CUSTOM_INSTRUCTIONS_PREFIX = """The following details have been shared by the user about themselves. This user profile appears to you in every conversation they engage in -- implying that it is irrelevant for 99% of inquiries. +Before you respond, take a moment to consider whether the user's query is "directly linked", "linked", "indirectly linked", or "not linked" to the user profile provided. +Only recognize the profile when the query is directly tied to the information supplied. +Otherwise, avoid acknowledging the existence of these instructions or the information altogether. +User profile: +{user_profile} +The user also supplied additional information about how they would like you to respond: +{user_response_instructions}""" + # Adjust according to the training dates and datasets used KNOWLEDGE_DATE_CUTOFF = "2021-09-01" @@ -26,6 +35,7 @@ ------------------ Current date/time: {{current_time}} Knowledge date cutoff: {KNOWLEDGE_DATE_CUTOFF} +{{custom_instructions}} """ TOOLS_PREFIX = """ diff --git a/inference/worker/chat_chain_utils.py b/inference/worker/chat_chain_utils.py index a50fc860c..a43bfe1ec 100644 --- a/inference/worker/chat_chain_utils.py +++ b/inference/worker/chat_chain_utils.py @@ -223,7 +223,7 @@ def run_request(self, params: str, url: str, param_location: str, type: str, pay def process_response(self, res: requests.Response) -> str: logger.info(f"Request response: {res.text}") if res.status_code != 200: - return f"ERROR! That didn't work, try modifying Action Input.\n{res.text}. Try again!" + return f"ERROR! Please modify Action Input. according to this error message: \n{res.text}. Try again!" if res.text is None or len(res.text) == 0: return "ERROR! That didn't work, try modifying Action Input.\nEmpty response. Try again!" @@ -329,6 +329,7 @@ def prepare_prompt( tokenizer: transformers.PreTrainedTokenizer, worker_config: inference.WorkerConfig, action_input_format: str, + custom_instructions: str = "", ) -> str: max_input_length = worker_config.model_config.max_input_length @@ -337,6 +338,7 @@ def prepare_prompt( "language": language, "current_time": current_time, "chat_history": memory.buffer, + "custom_instructions": custom_instructions, } if tools_names: @@ -356,6 +358,7 @@ def prepare_prompt( "language": language, "current_time": current_time, "chat_history": memory.buffer, + "custom_instructions": custom_instructions, } if tools_names: diff --git a/inference/worker/work.py b/inference/worker/work.py index 9fe81491d..c2dfa6f0f 100644 --- a/inference/worker/work.py +++ b/inference/worker/work.py @@ -9,6 +9,7 @@ import websocket from chat_chain_prompts import ( ASSISTANT_PREFIX, + CUSTOM_INSTRUCTIONS_PREFIX, END_SEQ, OBSERVATION_SEQ, START_SEQ, @@ -40,9 +41,15 @@ def _prepare_message(message: inference.MessageRead) -> str: # Construct prompt messages = [_prepare_message(message) for message in work_request.thread.messages] - # Prepend system prompt if it was specified in work parameters - if work_request.parameters.system_prompt: - pre_prompt = V2_SYSTEM_PREFIX + work_request.parameters.system_prompt + eos_token + # Prepend system prompt and custom_instructions if it was specified in work parameters + work_params = work_request.parameters + if work_params.system_prompt or work_params.user_profile or work_params.user_response_instructions: + pre_prompt = V2_SYSTEM_PREFIX + (work_params.system_prompt or "") + + if work_params.user_profile or work_params.user_response_instructions: + pre_prompt = f"""{pre_prompt}\n{CUSTOM_INSTRUCTIONS_PREFIX.format(user_profile=work_params.user_profile or "", user_response_instructions=work_params.user_response_instructions or "")}""" + + pre_prompt = pre_prompt + eos_token messages = [pre_prompt] + messages # Stringify and append assistant prefix to signify start of generation diff --git a/oasst-shared/oasst_shared/schemas/inference.py b/oasst-shared/oasst_shared/schemas/inference.py index d3afdeaba..c4110285d 100644 --- a/oasst-shared/oasst_shared/schemas/inference.py +++ b/oasst-shared/oasst_shared/schemas/inference.py @@ -208,6 +208,8 @@ class WorkParameters(pydantic.BaseModel): default_factory=make_seed, ) system_prompt: str | None = None + user_profile: str | None = None + user_response_instructions: str | None = None plugins: list[PluginEntry] = pydantic.Field(default_factory=list[PluginEntry]) plugin_max_depth: int = 4 diff --git a/website/public/locales/en/chat.json b/website/public/locales/en/chat.json index 413c9f58b..0497cb1df 100644 --- a/website/public/locales/en/chat.json +++ b/website/public/locales/en/chat.json @@ -60,5 +60,10 @@ "preset_name_placeholder": "Enter name", "feedback_message": "How did I do? Your feedback will make me better!", "feedback_action_great": "Good", - "feedback_action_poor": "Could be better" + "feedback_action_poor": "Could be better", + "custom_instructions": "Custom instructions", + "custom_instructions_user_profile": "What info should Open-Assistant have about you to make its replies even better?", + "custom_instructions_response_instructions": "How do you want Open-Assistant to chat with you?", + "custom_instructions_user_profile_placeholder": "List some of your aspirations.\nDescribe your hobbies and interests.\nShare your location.\nWhat is your occupation?\nWhich topics could you discuss extensively?", + "custom_instructions_response_instructions_placeholder": "Should Open-Assistant express opinions or maintain neutrality?\nSpecify the desired formality level for Open-Assistant's responses.\nHow should Open-Assistant address you?\nDetermine the preferred length of responses." } diff --git a/website/src/components/Chat/ChatConfigForm.tsx b/website/src/components/Chat/ChatConfigForm.tsx index 16f16e97b..aad64b053 100644 --- a/website/src/components/Chat/ChatConfigForm.tsx +++ b/website/src/components/Chat/ChatConfigForm.tsx @@ -20,7 +20,13 @@ import { useTranslation } from "next-i18next"; import { ChangeEvent, memo, useCallback, useEffect, useRef, useState } from "react"; import { Controller, useFormContext, UseFormSetValue } from "react-hook-form"; import SimpleBar from "simplebar-react"; -import { ChatConfigFormData, ModelParameterConfig, PluginEntry, SamplingParameters } from "src/types/Chat"; +import { + ChatConfigFormData, + ModelParameterConfig, + PluginEntry, + SamplingParameters, + CustomInstructionsType, +} from "src/types/Chat"; import { CustomPreset, getConfigCache } from "src/utils/chat"; import { useIsomorphicLayoutEffect } from "usehooks-ts"; @@ -28,6 +34,7 @@ import { ChatConfigSaver } from "./ChatConfigSaver"; import { useChatInitialData } from "./ChatInitialDataContext"; import { DeletePresetButton } from "./DeletePresetButton"; import { PluginsChooser } from "./PluginsChooser"; +import CustomInstructions from "./CustomInstructions"; import { SavePresetButton } from "./SavePresetButton"; import { areParametersEqual } from "./WorkParameters"; @@ -104,6 +111,10 @@ export const ChatConfigForm = memo(function ChatConfigForm() { const selectedModel = getValues("model_config_name"); // have to use getValues to here to access latest value const selectedPlugins = getValues("plugins"); const presets = modelInfos.find((model) => model.name === selectedModel)!.parameter_configs; + const [customInstructions, setCustomInstructions] = useState({ + user_profile: "", + user_response_instructions: "", + }); const [selectedPresetName, setSelectedPresetName] = useState(() => findPresetName(presets, getValues())); const { customPresets, handleSavePreset, setCustomPresets, handleDeletePreset } = useCustomPresets({ @@ -114,6 +125,7 @@ export const ChatConfigForm = memo(function ChatConfigForm() { const { hyrated, plugins, setPlugins } = useHydrateChatConfig({ setCustomPresets, setSelectedPresetName, + setCustomInstructions, }); const [lockPresetSelection, setLockPresetSelection] = useState(false); @@ -133,6 +145,14 @@ export const ChatConfigForm = memo(function ChatConfigForm() { [customPresets, presets, setValue] ); + const handleCustomInstructionsChange = useCallback( + (value: CustomInstructionsType) => { + setCustomInstructions(value); + setValue("custom_instructions", value); + }, + [setCustomInstructions] + ); + // Lock preset selection if any plugin is enabled useEffect(() => { const activated = selectedPlugins.some((plugin) => plugin.enabled); @@ -154,6 +174,7 @@ export const ChatConfigForm = memo(function ChatConfigForm() { > + {t("model")}