diff --git a/.github/workflows/deploy-to-node.yaml b/.github/workflows/deploy-to-node.yaml index a34e55033..f04484c6b 100644 --- a/.github/workflows/deploy-to-node.yaml +++ b/.github/workflows/deploy-to-node.yaml @@ -81,6 +81,7 @@ jobs: BACKEND_CORS_ORIGINS: ${{ vars.BACKEND_CORS_ORIGINS }} WEB_INFERENCE_SERVER_HOST: ${{ vars.WEB_INFERENCE_SERVER_HOST }} WEB_ENABLE_CHAT: ${{ vars.WEB_ENABLE_CHAT }} + WEB_BYE: ${{ vars.WEB_BYE }} WEB_ENABLE_DRAFTS_WITH_PLUGINS: ${{ vars.WEB_ENABLE_DRAFTS_WITH_PLUGINS }} WEB_NUM_GENERATED_DRAFTS: ${{ vars.WEB_NUM_GENERATED_DRAFTS }} WEB_CURRENT_ANNOUNCEMENT: ${{ vars.WEB_CURRENT_ANNOUNCEMENT }} diff --git a/README.md b/README.md index b0bb4562d..f8ed80496 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ +
+

:memo: NOTE: OpenAssistant is completed, and the project is now finished. Thank you to everyone who contributed! Check out our blog post for more information. The final published oasst2 dataset can be found on HuggingFace at OpenAssistant/oasst2

+
+
![GitHub Repo stars](https://img.shields.io/github/stars/LAION-AI/Open-Assistant?style=social) diff --git a/ansible/deploy-to-node.yaml b/ansible/deploy-to-node.yaml index f36e02071..33898bc19 100644 --- a/ansible/deploy-to-node.yaml +++ b/ansible/deploy-to-node.yaml @@ -284,6 +284,7 @@ INFERENCE_SERVER_API_KEY: "{{ lookup('ansible.builtin.env', 'WEB_INFERENCE_SERVER_API_KEY') }}" ENABLE_CHAT: "{{ lookup('ansible.builtin.env', 'WEB_ENABLE_CHAT') }}" + BYE: "{{ lookup('ansible.builtin.env', 'WEB_BYE') }}" ENABLE_DRAFTS_WITH_PLUGINS: "{{ lookup('ansible.builtin.env', 'WEB_ENABLE_DRAFTS_WITH_PLUGINS')}}" diff --git a/backend/requirements.txt b/backend/requirements.txt index b5cf3ca99..79765cd53 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -10,8 +10,8 @@ loguru==0.6.0 numpy>=1.23.2 prometheus-fastapi-instrumentator==5.9.1 psycopg2-binary==2.9.5 -pydantic==1.10.4 -pydantic[email]==1.10.4 +pydantic==1.10.7 +pydantic[email]==1.10.7 python-dotenv==0.21.0 python-jose[cryptography]==3.3.0 redis==4.5.5 diff --git a/data/datasets/__init__.py b/data/datasets/__init__.py index 6cf9e7f02..7b2c077ea 100644 --- a/data/datasets/__init__.py +++ b/data/datasets/__init__.py @@ -4,6 +4,8 @@ "tv_dialogue": "sedthh/tv_dialogue", # TV and Movie dialogues and transcripts "fd_dialogue": "sedthh/fd_dialogue", # TV and Movie dialogues and transcripts from ForeverDreaming "tlcv2.0_oa": "pythainlp/tlcv2.0_oa", # Thai classical literature texts + "fa-isna-news": "pourmand1376/isna-news", # Isna Persian News + "fa-wikipedia": "pourmand1376/fa-wikipedia", # Farsi Wikipedia texts } INSTRUCTION_DATASETS = { diff --git a/data/datasets/fa-isna-news/README.md b/data/datasets/fa-isna-news/README.md new file mode 100644 index 000000000..fa216bef4 --- /dev/null +++ b/data/datasets/fa-isna-news/README.md @@ -0,0 +1,2 @@ +This text-only dataset is crawled from [Isna news](https://isna.ir/). This is +biggest farsi news agency and thus the text is pretty clean. diff --git a/data/datasets/fa-wikipedia/README.md b/data/datasets/fa-wikipedia/README.md new file mode 100644 index 000000000..728190a73 --- /dev/null +++ b/data/datasets/fa-wikipedia/README.md @@ -0,0 +1,6 @@ +This dataset is crawled from +[farsi wikipedia](https://fa.wikipedia.org/wiki/%D8%B5%D9%81%D8%AD%D9%87%D9%94_%D8%A7%D8%B5%D9%84%DB%8C). +This is valuable clean text data in persian (Farsi). It contains information +about all subjects. + +It has 2.53M Articles. diff --git a/docs/blog/2023-02-11-architecture.md b/docs/blog/2023-02-11-architecture.md index 4eb026af7..434a32ee6 100644 --- a/docs/blog/2023-02-11-architecture.md +++ b/docs/blog/2023-02-11-architecture.md @@ -10,7 +10,7 @@ I thought it would make sense to document our architecture as a blog post, since it would be easily visible for everyone, and it would also be nice to look back a couple months from now and see how stuff has changed :) -A lot of work was done in the past couple of week to establish the needed +A lot of work was done in the past couple of weeks to establish the needed infrastructure both for data collection and inference purposes. Thanks to everyone, and especially community members who supported us so far! diff --git a/docs/blog/2023-10-25-open-assistant-is-completed/index.mdx b/docs/blog/2023-10-25-open-assistant-is-completed/index.mdx new file mode 100644 index 000000000..f650c1b3c --- /dev/null +++ b/docs/blog/2023-10-25-open-assistant-is-completed/index.mdx @@ -0,0 +1,20 @@ +--- +title: OpenAssistant is Completed! +description: OpenAssistant is Completed! +authors: [yk] +tags: [open-assistant, youtube] +image: https://img.youtube.com/vi/gqtmUHhaplo/0.jpg +--- + +import ReactPlayer from "react-player"; + + + +The final published oasst2 dataset can be found on HuggingFace at +[OpenAssistant/oasst2](https://huggingface.co/datasets/OpenAssistant/oasst2). + + diff --git a/docs/docs/faq.md b/docs/docs/faq.md index 91c277cb6..8566b9dfe 100644 --- a/docs/docs/faq.md +++ b/docs/docs/faq.md @@ -15,12 +15,12 @@ In this page, there are some of the most frequently asked questions. -We have released candidate supervised finetuning (SFT) models using both Pythia -and LLaMa, as well as candidate reward models for reinforcement learning from -human feedback training using Pythia, which you can try, and are beginning the -process of applying (RLHF). We have also released the first version of the -OpenAssistant Conversations dataset -[here](https://huggingface.co/datasets/OpenAssistant/oasst1). +This project has concluded. We have released supervised finetuning (SFT) models +using Llama 2, LLaMa, Falcon, Pythia, and StabeLM as well as reinforcement +learning from human feedback trained models and reward models, all of which are +available at [here](https://huggingface.co/OpenAssistant). In addition to our +models, we have released three datasets from OpenAssistant conversations, and a +[research paper](https://arxiv.org/abs/2304.07327). @@ -31,9 +31,8 @@ OpenAssistant Conversations dataset -You can play with our best candidate model -[here](https://open-assistant.io/chat) and provide thumbs up/down responses to -help us improve the model in future! +Our online demonstration is no longer available, but the models remain available +to download [here](https://huggingface.co/OpenAssistant). @@ -44,37 +43,18 @@ help us improve the model in future! -The candidate Pythia SFT models are +All of our models are [available on HuggingFace](https://huggingface.co/OpenAssistant) and can be -loaded via the HuggingFace Transformers library. As such you may be able to use -them with sufficient hardware. There are also spaces on HF which can be used to -chat with the OA candidate without your own hardware. However, these models are -not final and can produce poor or undesirable outputs. +loaded via the HuggingFace Transformers library or other runners if converted. +As such you may be able to use them with sufficient hardware. There are also +spaces on HF which can be used to chat with the OA candidate without your own +hardware. However, some of these models are not final and can produce poor or +undesirable outputs. -LLaMa SFT models cannot be released directly due to Meta's license but XOR +LLaMa (v1) SFT models cannot be released directly due to Meta's license but XOR weights are released on the HuggingFace org. Follow the process in the README -there to obtain a full model from these XOR weights. - - - -
- - -### Is there an API available? - - - -There is no API currently available for Open Assistant. Any mention of an API in -documentation is referencing the website's internal API. We understand that an -API is a highly requested feature, but unfortunately, we can't provide one at -this time due to a couple of reasons. Firstly, the inference system is already -under high load and running off of compute from our sponsors. Secondly, the -project's primary goal is currently data collection and model training, not -providing a product. - -However, if you're looking to run inference, you can host the model yourself -either on your own hardware or with a cloud provider. We appreciate your -understanding and patience as we continue to develop this project. +there to obtain a full model from these XOR weights. Llama 2 models are not +required to be XORed.
@@ -102,15 +82,13 @@ inference setup and UI locally unless you wish to assist in development. All Open Assistant code is licensed under Apache 2.0. This means it is available for a wide range of uses including commercial use. -The Open Assistant Pythia based models are released as full weights and will be -licensed under the Apache 2.0 license. - -The Open Assistant LLaMa based models will be released only as delta weights -meaning you will need the original LLaMa weights to use them, and the license -restrictions will therefore be those placed on the LLaMa weights. +Open Assistant models are released under the license of their respective base +models, be that Llama 2, Falcon, Pythia, or StableLM. LLaMa (not 2) models are +only released as XOR weights, meaning you will need the original LLaMa weights +to use them. -The Open Assistant data is released under a Creative Commons license allowing a -wide range of uses including commercial use. +The Open Assistant data is released under Apache-2.0 allowing a wide range of +uses including commercial use. @@ -138,9 +116,8 @@ you to everyone who has taken part! -The model code, weights, and data are free. We are additionally hosting a free -public instance of our best current model for as long as we can thanks to -compute donation from Stability AI via LAION! +The model code, weights, and data are free. Our free public instance of our best +models is not longer available due to the project's conclusion. @@ -151,10 +128,9 @@ compute donation from Stability AI via LAION! -The current smallest (Pythia) model is 12B parameters and is challenging to run -on consumer hardware, but can run on a single professional GPU. In future there -may be smaller models and we hope to make progress on methods like integer -quantisation which can help run the model on smaller hardware. +The current smallest models are 7B parameters and are challenging to run on +consumer hardware, but can run on a single professional GPU or be quantized to +run on more widely available hardware. @@ -165,13 +141,7 @@ quantisation which can help run the model on smaller hardware. -If you want to help in the data collection for training the model, go to the -website [https://open-assistant.io/](https://open-assistant.io/). - -If you want to contribute code, take a look at the -[tasks in GitHub](https://github.com/orgs/LAION-AI/projects/3) and comment on an -issue stating your wish to be assigned. You can also take a look at this -[contributing guide](https://github.com/LAION-AI/Open-Assistant/blob/main/CONTRIBUTING.md). +This project has now concluded. @@ -190,104 +160,6 @@ well as accelerate, DeepSpeed, bitsandbytes, NLTK, and other libraries. -## Questions about the data collection website - -
- - -### Can I use ChatGPT to help in training Open Assistant, for instance, by generating answers? - - - -No, it is against their terms of service to use it to help train other models. -See -[this issue](https://github.com/LAION-AI/Open-Assistant/issues/471#issuecomment-1374392299). -ChatGPT-like answers will be removed. - -
- -
- - -### What should I do if I don't know how to complete the task as an assistant? - - -Skip it. -
- -
- - -### Should I fact check the answers by the assistant? - - - -Yes, you should try. If you are not sure, skip the task. - -
- -
- - -### How can I see my score? - - - -In your [account settings](https://open-assistant.io/account). - -
- -
- - -### Can we see how many data points have been collected? - - - -You can see a regularly updated interface at -[https://open-assistant.io/stats](https://open-assistant.io/stats). - -
- -
- - -### How do I write and label prompts? - - - -Check the -[guidelines](https://projects.laion.ai/Open-Assistant/docs/guides/guidelines). - -
- -
- - -### Where can I report a bug or create a new feature request? - - - -In the [GitHub issues](https://github.com/LAION-AI/Open-Assistant/issues). - -
- -
- - -### Why am I not allowed to write about this topic, even though it isn't illegal? - - - -We want to ensure that the Open Assistant dataset is as accessible as possible. -As such, it's necessary to avoid any harmful or offensive content that could be -grounds for removal on sites such as Hugging Face. Likewise, we want the model -to be trained to reject as few questions as possible, so it's important to not -include prompts that leave the assistant with no other choice but to refuse in -order to avoid the generation of harmful content. - -
- ## Questions about the development process
diff --git a/docs/docs/intro.md b/docs/docs/intro.md index 326502bfe..98f50762c 100644 --- a/docs/docs/intro.md +++ b/docs/docs/intro.md @@ -1,3 +1,9 @@ +# Notice + +**Open Assistant has now concluded.** Please see +[this video](https://www.youtube.com/watch?v=gqtmUHhaplo) for more information. +Thanks you to all those who made this project possible. + # Introduction > The FAQ page is available at diff --git a/inference/server/oasst_inference_server/routes/chats.py b/inference/server/oasst_inference_server/routes/chats.py index 8b151074b..1959b094e 100644 --- a/inference/server/oasst_inference_server/routes/chats.py +++ b/inference/server/oasst_inference_server/routes/chats.py @@ -153,6 +153,8 @@ async def create_assistant_message( system_prompt=request.system_prompt, plugins=request.plugins, plugin_max_depth=settings.plugin_max_depth, + user_profile=request.user_profile, + user_response_instructions=request.user_response_instructions, ) assistant_message = await ucr.initiate_assistant_message( parent_id=request.parent_id, diff --git a/inference/server/oasst_inference_server/routes/workers.py b/inference/server/oasst_inference_server/routes/workers.py index 20195f9b2..656262496 100644 --- a/inference/server/oasst_inference_server/routes/workers.py +++ b/inference/server/oasst_inference_server/routes/workers.py @@ -1,7 +1,6 @@ import asyncio import datetime -from enum import Enum -from typing import Awaitable, NamedTuple, TypeVar, cast +from typing import cast import fastapi import pydantic @@ -44,6 +43,20 @@ def __init__( self.original_exception = original_exception +async def add_worker_connect_event( + session: database.AsyncSession, + worker_id: str, + worker_info: inference.WorkerInfo, +): + event = models.DbWorkerEvent( + worker_id=worker_id, + event_type=models.WorkerEventType.connect, + worker_info=worker_info, + ) + session.add(event) + await session.commit() + + class WorkRequestContainer(pydantic.BaseModel): work_request: inference.WorkRequest message_id: str @@ -72,321 +85,191 @@ def get_work_request_container(work_request_map: WorkRequestContainerMap, reques return container -class HandleWorkerContext(NamedTuple): - """ - Whenever a worker connects to the inference server over a websocket, we create an instance of this class to hold all - state that is associated with that worker and connection. - """ - - websocket: fastapi.WebSocket - worker_id: str - worker_info: inference.WorkerInfo +@router.websocket("/work") +async def handle_worker( + websocket: fastapi.WebSocket, + api_key: str = worker_utils.api_key_header, + protocol_version: str = worker_utils.protocol_version_header, +): + await websocket.accept() - work_queue: queueing.RedisQueue + try: + worker_utils.get_protocol_version(protocol_version) + api_key = worker_utils.get_api_key(api_key) + worker_id = await worker_utils.get_worker_id(api_key=api_key, protocol_version=protocol_version) + except fastapi.HTTPException as e: + logger.warning(f"handle_worker: {e.status_code=} {e.detail=}") + if e.status_code == fastapi.status.HTTP_426_UPGRADE_REQUIRED: + await worker_utils.send_worker_request(websocket=websocket, request=inference.UpgradeProtocolRequest()) + elif e.status_code == fastapi.status.HTTP_401_UNAUTHORIZED: + await worker_utils.send_worker_request(websocket=websocket, request=inference.WrongApiKeyRequest()) + try: + await websocket.close(code=e.status_code, reason=e.detail) + except Exception: + pass + raise fastapi.WebSocketException(e.status_code, e.detail) + + logger.info(f"handle_worker: {worker_id=}") + worker_info = await worker_utils.receive_worker_info(websocket) + logger.info(f"handle_worker: {worker_info=}") + worker_config = worker_info.config + worker_compat_hash = worker_config.compat_hash + work_queue = queueing.work_queue(deps.redis_client, worker_compat_hash) + redis_client = deps.make_redis_client() + blocking_work_queue = queueing.work_queue(redis_client, worker_compat_hash) + worker_session = worker_utils.WorkerSession( + worker_id=worker_id, + worker_info=worker_info, + ) + work_request_map: dict[str, WorkRequestContainer] = {} + pending_futures = set() + try: + async with deps.manual_create_session() as session: + await add_worker_connect_event(session=session, worker_id=worker_id, worker_info=worker_info) + await worker_utils.store_worker_session(worker_session) - blocking_work_queue: queueing.RedisQueue + async def _update_session(metrics: inference.WorkerMetricsInfo): + worker_session.requests_in_flight = len(work_request_map) + if metrics: + worker_session.metrics = metrics + await worker_utils.store_worker_session(worker_session) - work_request_map: dict[str, WorkRequestContainer] = dict() + def _add_dequeue(ftrs: set): + requests_in_progress = len(work_request_map) + if requests_in_progress < worker_config.max_parallel_requests: + ftrs.add(asyncio.ensure_future(blocking_work_queue.dequeue(timeout=0))) - @property - def config(self) -> inference.WorkerConfig: - return self.worker_info.config + def _add_receive(ftrs: set): + ftrs.add(asyncio.ensure_future(worker_utils.receive_worker_response(websocket=websocket))) - @property - def worker_compat_hash(self) -> str: - return self.config.compat_hash + _add_dequeue(pending_futures) + _add_receive(pending_futures) - @property - def num_requests_in_progress(self) -> int: - return len(self.work_request_map) + logger.info(f"handle_worker: {worker_id=} started") + while True: + if websocket.client_state == fastapi.websockets.WebSocketState.DISCONNECTED: + raise WorkerDisconnectException("Worker disconnected") + (done, pending_futures) = await asyncio.wait( + pending_futures, timeout=settings.worker_ping_interval, return_when=asyncio.FIRST_COMPLETED + ) + ftr: asyncio.Future + for ftr in done: + result = ftr.result() + if result is None: + logger.error(f"handle_worker: {worker_id=} received None from queue. This should never happen.") + raise RuntimeError("Received None from queue. This should never happen.") + elif isinstance(result, tuple): + try: + _, message_id = result + work_request = await initiate_work_for_message( + websocket=websocket, + work_queue=work_queue, + message_id=message_id, + worker_id=worker_id, + worker_config=worker_config, + ) + work_request_map[work_request.id] = WorkRequestContainer( + work_request=work_request, message_id=message_id + ) + except chat_schema.MessageCancelledException as e: + logger.warning(f"Message was cancelled before work could be initiated: {e.message_id=}") + except chat_schema.MessageTimeoutException as e: + logger.warning(f"Message timed out before work could be initiated: {e.message.id=}") + await handle_timeout(message=e.message) + finally: + _add_dequeue(pending_futures) + else: + try: + worker_response: inference.WorkerResponse = result + match worker_response.response_type: + case "pong": + worker_response = cast(inference.PongResponse, worker_response) + await _update_session(worker_response.metrics) + case "token": + worker_response = cast(inference.TokenResponse, worker_response) + await handle_token_response( + work_request_map=work_request_map, + response=worker_response, + ) + case "generated_text": + worker_response = cast(inference.GeneratedTextResponse, worker_response) + await handle_generated_text_response( + work_request_map=work_request_map, + response=worker_response, + ) + await _update_session(worker_response.metrics) + case "error": + worker_response = cast(inference.ErrorResponse, worker_response) + await handle_error_response( + work_request_map=work_request_map, + response=worker_response, + ) + await _update_session(worker_response.metrics) + case "general_error": + worker_response = cast(inference.GeneralErrorResponse, worker_response) + await handle_general_error_response( + response=worker_response, + ) + await _update_session(worker_response.metrics) + case "safe_prompt": + logger.info("Received safe prompt response") + worker_response = cast(inference.SafePromptResponse, worker_response) + await handle_safe_prompt_response( + response=worker_response, + work_request_map=work_request_map, + ) + case "plugin_intermediate": + worker_response = cast(inference.PluginIntermediateResponse, worker_response) + await handle_plugin_intermediate_response( + work_request_map=work_request_map, + response=worker_response, + ) + case _: + raise RuntimeError(f"Unknown response type: {worker_response.response_type}") + finally: + if len(pending_futures) == 0: + _add_dequeue(pending_futures) + _add_receive(pending_futures) + if not done: + await worker_utils.send_worker_request(websocket, inference.PingRequest()) - @classmethod - async def build(cls, websocket: fastapi.WebSocket, api_key: str, protocol_version: str): - try: - worker_utils.get_protocol_version(protocol_version) - api_key = worker_utils.get_api_key(api_key) - worker_id = await worker_utils.get_worker_id(api_key=api_key, protocol_version=protocol_version) - except fastapi.HTTPException as e: - logger.warning(f"handle_worker: {e.status_code=} {e.detail=}") - if e.status_code == fastapi.status.HTTP_426_UPGRADE_REQUIRED: - await worker_utils.send_worker_request(websocket=websocket, request=inference.UpgradeProtocolRequest()) - elif e.status_code == fastapi.status.HTTP_401_UNAUTHORIZED: - await worker_utils.send_worker_request(websocket=websocket, request=inference.WrongApiKeyRequest()) + except Exception as e: + logger.exception(f"Error while handling worker {worker_id}: {str(e)}") + logger.info(f"Handling {len(work_request_map)} work requests outstanding") + for container in work_request_map.values(): try: - await websocket.close(code=e.status_code, reason=e.detail) - except Exception: - pass - raise fastapi.WebSocketException(e.status_code, e.detail) - - logger.info(f"handle_worker: {worker_id=}") - worker_info = await worker_utils.receive_worker_info(websocket) - logger.info(f"handle_worker: {worker_info=}") - - work_queue = queueing.work_queue(deps.redis_client, worker_info.config.compat_hash) - - # Make a new Redis client that we can use for the blocking requests without affecting the non-blocking calls - redis_client = deps.make_redis_client() - blocking_work_queue = queueing.work_queue(redis_client, worker_info.config.compat_hash) - - return cls( - websocket=websocket, - worker_id=worker_id, - worker_info=worker_info, - work_queue=work_queue, - blocking_work_queue=blocking_work_queue, - ) - - async def close(self): + message_id = container.message_id + if container.num_responses == 0: + logger.warning(f"Marking {message_id=} as pending since no work was done.") + async with deps.manual_chat_repository() as cr: + await cr.reset_work(message_id) + await work_queue.enqueue(message_id, enforce_max_size=False) + else: + logger.warning(f"Aborting {message_id=}") + await abort_message(message_id=message_id, error="Aborted due to worker error.") + except Exception as e: + logger.exception(f"Error while trying to reset work for {message_id=}: {str(e)}") + finally: + logger.info(f"Worker {worker_id} disconnected") try: - await self.blocking_work_queue.redis_client.close() + await redis_client.close() except Exception: logger.warning("Error while closing redis client") - try: - await self.websocket.close() + await worker_utils.delete_worker_session(worker_session.id) except Exception: - logger.warning("Error while closing websocket") - - -class FutureType(str, Enum): - WORK_REQUEST = "WORK_REQUEST" - WORKER_RESPONSE = "WORKER_RESPONSE" - - -AwaitResult = TypeVar("AwaitResult") - - -class FuturesManager: - _context: HandleWorkerContext - _futures: set[asyncio.Future] - - def __init__(self, context: HandleWorkerContext): - self._context = context - self._futures = set() - - def ensure_listening_for_work_requests(self): - requests_in_progress = self._context.num_requests_in_progress - if requests_in_progress < self._context.config.max_parallel_requests: - future = self._as_typed_future( - FutureType.WORK_REQUEST, self._context.blocking_work_queue.dequeue(timeout=0) - ) - self._futures.add(future) - - def ensure_listening_to_worker_responses(self): - future = self._as_typed_future( - FutureType.WORKER_RESPONSE, worker_utils.receive_worker_response(websocket=self._context.websocket) - ) - self._futures.add(future) - - async def wait_for_event_or_timeout(self): - completed_futures, pending_futures = await asyncio.wait( - self._futures, timeout=settings.worker_ping_interval, return_when=asyncio.FIRST_COMPLETED - ) - self._futures = pending_futures - - return [future.result() for future in completed_futures] - - def cancel_all(self): - logger.info(f"Cancelling {len(self._futures)} pending futures") - for ftr in self._futures: + logger.warning("Error while deleting worker session") + # try closing websocket if it's still open + logger.info(f"Cancelling {len(pending_futures)} pending futures") + for ftr in pending_futures: try: ftr.cancel() except Exception: logger.warning("Error while cancelling pending future") - - def _as_typed_future(self, future_type: FutureType, awaitable: Awaitable[AwaitResult]) -> asyncio.Future: - async def wrapper(): - result = await awaitable - return future_type, result - - # Note: important to call the wrapper to create the coroutine - return asyncio.ensure_future(wrapper()) - - -class SessionManager: - _context: HandleWorkerContext - _session: worker_utils.WorkerSession - - def __init__(self, context: HandleWorkerContext): - self._context = context - self._session = worker_utils.WorkerSession( - worker_id=context.worker_id, - worker_info=context.worker_info, - ) - - async def update(self, metrics: inference.WorkerMetricsInfo | None = None): - self._session.requests_in_flight = len(self._context.work_request_map) - if metrics: - self._session.metrics = metrics - await worker_utils.store_worker_session(self._session) - - async def _add_worker_connect_event( - self, - session: database.AsyncSession, - worker_id: str, - worker_info: inference.WorkerInfo, - ): - event = models.DbWorkerEvent( - worker_id=worker_id, - event_type=models.WorkerEventType.connect, - worker_info=worker_info, - ) - session.add(event) - await session.commit() - - async def init(self): - async with deps.manual_create_session() as session: - await self._add_worker_connect_event( - session=session, worker_id=self._context.worker_id, worker_info=self._context.worker_info - ) - await worker_utils.store_worker_session(self._session) - - async def close(self): try: - await worker_utils.delete_worker_session(self._session.id) + await websocket.close() except Exception: - logger.warning("Error while deleting worker session") - - -async def _handle_work_request(message_id: str, context: HandleWorkerContext, futures: FuturesManager): - try: - work_request = await initiate_work_for_message( - websocket=context.websocket, - work_queue=context.work_queue, - message_id=message_id, - worker_id=context.worker_id, - worker_config=context.config, - ) - context.work_request_map[work_request.id] = WorkRequestContainer( - work_request=work_request, message_id=message_id - ) - except chat_schema.MessageCancelledException as e: - logger.warning(f"Message was cancelled before work could be initiated: {e.message_id=}") - except chat_schema.MessageTimeoutException as e: - logger.warning(f"Message timed out before work could be initiated: {e.message.id=}") - await handle_timeout(message=e.message) - finally: - futures.ensure_listening_for_work_requests() - - -async def _handle_worker_response( - worker_response: inference.WorkerResponse, - context: HandleWorkerContext, - futures: FuturesManager, - session: SessionManager, -): - try: - match worker_response.response_type: - case "pong": - worker_response = cast(inference.PongResponse, worker_response) - await session.update(worker_response.metrics) - case "token": - worker_response = cast(inference.TokenResponse, worker_response) - await handle_token_response( - work_request_map=context.work_request_map, - response=worker_response, - ) - case "generated_text": - worker_response = cast(inference.GeneratedTextResponse, worker_response) - await handle_generated_text_response( - work_request_map=context.work_request_map, - response=worker_response, - ) - await session.update(worker_response.metrics) - case "error": - worker_response = cast(inference.ErrorResponse, worker_response) - await handle_error_response( - work_request_map=context.work_request_map, - response=worker_response, - ) - await session.update(worker_response.metrics) - case "general_error": - worker_response = cast(inference.GeneralErrorResponse, worker_response) - await handle_general_error_response( - response=worker_response, - ) - await session.update(worker_response.metrics) - case "safe_prompt": - logger.info("Received safe prompt response") - worker_response = cast(inference.SafePromptResponse, worker_response) - await handle_safe_prompt_response( - response=worker_response, - work_request_map=context.work_request_map, - ) - case "plugin_intermediate": - worker_response = cast(inference.PluginIntermediateResponse, worker_response) - await handle_plugin_intermediate_response( - work_request_map=context.work_request_map, - response=worker_response, - ) - case _: - raise RuntimeError(f"Unknown response type: {worker_response.response_type}") - finally: - if len(futures._futures) == 0: - futures.ensure_listening_for_work_requests() - futures.ensure_listening_to_worker_responses() - - -async def _reset_or_abort_open_work_requests(context: HandleWorkerContext): - logger.info(f"Handling {context.num_requests_in_progress} work requests outstanding") - for container in context.work_request_map.values(): - try: - message_id = container.message_id - if container.num_responses == 0: - logger.warning(f"Marking {message_id=} as pending since no work was done.") - async with deps.manual_chat_repository() as cr: - await cr.reset_work(message_id) - await context.work_queue.enqueue(message_id, enforce_max_size=False) - else: - logger.warning(f"Aborting {message_id=}") - await abort_message(message_id=message_id, error="Aborted due to worker error.") - except Exception as e: - logger.exception(f"Error while trying to reset work for {message_id=}: {str(e)}") - - -@router.websocket("/work") -async def handle_worker( - websocket: fastapi.WebSocket, - api_key: str = worker_utils.api_key_header, - protocol_version: str = worker_utils.protocol_version_header, -): - await websocket.accept() - - try: - context = await HandleWorkerContext.build(websocket, api_key, protocol_version) - futures = FuturesManager(context) - session = SessionManager(context) - await session.init() - futures.ensure_listening_for_work_requests() - futures.ensure_listening_to_worker_responses() - - logger.info(f"handle_worker: {context.worker_id=} started") - while True: - if websocket.client_state == fastapi.websockets.WebSocketState.DISCONNECTED: - raise WorkerDisconnectException("Worker disconnected") - - results = await futures.wait_for_event_or_timeout() - for type_, result in results: - if type_ == FutureType.WORK_REQUEST: - _, message_id = result - await _handle_work_request(message_id, context, futures) - elif type_ == FutureType.WORKER_RESPONSE: - worker_response = cast(inference.WorkerResponse, result) - await _handle_worker_response(worker_response, context, futures, session) - else: - raise ValueError(f"Unexpected future type: {type_}") - - if len(results) == 0: - await worker_utils.send_worker_request(websocket, inference.PingRequest()) - - except Exception as e: - logger.exception(f"Error while handling worker {context.worker_id}: {str(e)}") - await _reset_or_abort_open_work_requests(context) - finally: - logger.info(f"Worker {context.worker_id} disconnected") - await session.close() - futures.cancel_all() - await context.close() + logger.warning("Error while closing websocket") @router.get("/sessions") diff --git a/inference/server/oasst_inference_server/schemas/chat.py b/inference/server/oasst_inference_server/schemas/chat.py index 74ec4a09a..c8a266ef7 100644 --- a/inference/server/oasst_inference_server/schemas/chat.py +++ b/inference/server/oasst_inference_server/schemas/chat.py @@ -15,6 +15,8 @@ class CreateAssistantMessageRequest(pydantic.BaseModel): model_config_name: str sampling_parameters: inference.SamplingParameters = pydantic.Field(default_factory=inference.SamplingParameters) system_prompt: str | None = None + user_profile: str | None = None + user_response_instructions: str | None = None plugins: list[inference.PluginEntry] = pydantic.Field(default_factory=list[inference.PluginEntry]) used_plugin: inference.PluginUsed | None = None diff --git a/inference/server/oasst_inference_server/worker_utils.py b/inference/server/oasst_inference_server/worker_utils.py index f793964bf..82fb64545 100644 --- a/inference/server/oasst_inference_server/worker_utils.py +++ b/inference/server/oasst_inference_server/worker_utils.py @@ -53,7 +53,7 @@ def get_protocol_version(protocol_version: str = protocol_version_header) -> str async def get_worker_id( api_key: str = Depends(get_api_key), protocol_version: str = Depends(get_protocol_version), -) -> str: +) -> models.DbWorker: """Get the ID of a worker from its API key and protocol version.""" logger.info(f"get_worker: {api_key=}, {protocol_version=}") query = sqlmodel.select(models.DbWorker).where(models.DbWorker.api_key == api_key) diff --git a/inference/worker/__main__.py b/inference/worker/__main__.py index 569e34027..cb7d3089a 100644 --- a/inference/worker/__main__.py +++ b/inference/worker/__main__.py @@ -34,7 +34,7 @@ def main(): tokenizer = None else: tokenizer: transformers.PreTrainedTokenizer = transformers.AutoTokenizer.from_pretrained(model_config.model_id) - logger.warning(f"Tokenizer {tokenizer.name_or_path} vocab size: {tokenizer.vocab_size}") + logger.warning(f"Tokenizer {tokenizer.name_or_path} vocab size: {len(tokenizer)}") inference_http = utils.HttpClient( base_url=settings.inference_server_url, diff --git a/inference/worker/basic_hf_server.py b/inference/worker/basic_hf_server.py index b97fb8d83..052680f01 100644 --- a/inference/worker/basic_hf_server.py +++ b/inference/worker/basic_hf_server.py @@ -138,7 +138,7 @@ def load_models(): hf_config = transformers.AutoConfig.from_pretrained(model_config.model_id) logger.warning(f"Loading model {model_config.model_id}...") tokenizer = transformers.AutoTokenizer.from_pretrained(model_config.model_id) - logger.warning(f"tokenizer {tokenizer.name_or_path} has vocab size {tokenizer.vocab_size}") + logger.warning(f"tokenizer {tokenizer.name_or_path} has vocab size {len(tokenizer)}") # see `decode_token` method, taken from HF text-generation-inference tokenizer.add_special_tokens({"additional_special_tokens": [""]}) diff --git a/inference/worker/chat_chain.py b/inference/worker/chat_chain.py index c4a540697..78dadcb12 100644 --- a/inference/worker/chat_chain.py +++ b/inference/worker/chat_chain.py @@ -6,6 +6,7 @@ import websocket from chat_chain_prompts import ( ASSISTANT_PREFIX, + CUSTOM_INSTRUCTIONS_PREFIX, HUMAN_PREFIX, JSON_FORMAT_NO_PAYLOAD, JSON_FORMAT_PAYLOAD, @@ -13,8 +14,6 @@ PREFIX, SUFFIX, THOUGHT_SEQ, - V2_ASST_PREFIX, - V2_PROMPTER_PREFIX, ) from chat_chain_utils import compose_tools_from_plugin, extract_tool_and_input, prepare_prompt, use_tool from hf_langchain_inference import HFInference @@ -25,6 +24,7 @@ from oasst_shared.model_configs import ModelConfig from oasst_shared.schemas import inference from settings import settings +from utils import special_tokens # Exclude tools description from final prompt. Saves ctx space but can hurt output # quality especially if truncation kicks in. Dependent on model used @@ -56,6 +56,7 @@ def __init__( tool_names: list[str], language: str, action_input_format: str, + custom_instructions: str = "", ): self.tokenizer = tokenizer self.worker_config = worker_config @@ -66,6 +67,7 @@ def __init__( self.language = language self.action_input_format = action_input_format self.current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + self.custom_instructions = custom_instructions def call(self, prompt: str) -> tuple[str, str]: """Prepares and truncates prompt, calls LLM, returns used prompt and response.""" @@ -79,6 +81,7 @@ def call(self, prompt: str) -> tuple[str, str]: self.tokenizer, self.worker_config, self.action_input_format, + self.custom_instructions, ) # We do not strip() outputs as it seems to degrade instruction-following abilities of the model @@ -111,6 +114,7 @@ def handle_plugin_usage( plugin_max_depth: int, ws: websocket.WebSocket, work_request_id: str, + custom_instructions: str = "", ) -> tuple[str, inference.PluginUsed]: execution_details = inference.PluginExecutionDetails( inner_monologue=[], @@ -138,11 +142,23 @@ def handle_plugin_usage( action_input_format = ( JSON_FORMAT_PAYLOAD if prompt_template.template.find("payload") != -1 else JSON_FORMAT_NO_PAYLOAD ) - eos_token = tokenizer.eos_token if hasattr(tokenizer, "eos_token") else "" + eos_token = "" + if special_tokens["end"]: + eos_token = special_tokens["end"] + elif hasattr(tokenizer, "eos_token"): + eos_token = tokenizer.eos_token tool_names = [tool.name for tool in tools] chain = PromptedLLM( - tokenizer, worker_config, parameters, prompt_template, memory, tool_names, language, action_input_format + tokenizer, + worker_config, + parameters, + prompt_template, + memory, + tool_names, + language, + action_input_format, + custom_instructions, ) # send "thinking..." intermediate step to UI (This will discard queue position 0) immediately @@ -157,7 +173,7 @@ def handle_plugin_usage( ), ) - init_prompt = f"{input_prompt}{eos_token}{V2_ASST_PREFIX}" + init_prompt = f"{input_prompt}{eos_token}{special_tokens['assistant']}" init_prompt, chain_response = chain.call(init_prompt) inner_monologue.append("In: " + str(init_prompt)) @@ -190,7 +206,9 @@ def handle_plugin_usage( # Save previous chain response for use in final prompt prev_chain_response = chain_response - new_prompt = f"{input_prompt}{eos_token}{V2_ASST_PREFIX}{chain_response}{OBSERVATION_SEQ} {tool_response}" + new_prompt = ( + f"{input_prompt}{eos_token}{special_tokens['assistant']}{chain_response}{OBSERVATION_SEQ} {tool_response}" + ) new_prompt, chain_response = chain.call(new_prompt) @@ -226,15 +244,13 @@ def handle_plugin_usage( chain_finished = True if REMOVE_TOOLS_FROM_FINAL_PROMPT: - TEMPLATE = f"""{V2_PROMPTER_PREFIX}{PREFIX}{SUFFIX}""" + TEMPLATE = f"""{special_tokens['prompter']}{PREFIX}{SUFFIX}""" input_variables = ["input", "chat_history", "language", "current_time"] prompt_template = PromptTemplate(input_variables=input_variables, template=TEMPLATE) tool_names = None - final_input = ( - f"{input_prompt}{eos_token}{V2_ASST_PREFIX}\n{prev_chain_response}{OBSERVATION_SEQ} {tool_response}" - ) + final_input = f"{input_prompt}{eos_token}{special_tokens['assistant']}\n{prev_chain_response}{OBSERVATION_SEQ} {tool_response}" inner_prompt = prepare_prompt( final_input, prompt_template, @@ -245,6 +261,7 @@ def handle_plugin_usage( tokenizer, worker_config, action_input_format, + custom_instructions, ) inner_prompt = f"{inner_prompt}\n{THOUGHT_SEQ} I now know the final answer\n{ASSISTANT_PREFIX}: " @@ -296,17 +313,31 @@ def handle_standard_usage( memory: ConversationBufferMemory, worker_config: inference.WorkerConfig, tokenizer: transformers.PreTrainedTokenizer, + custom_instructions: str = "", ): - eos_token = tokenizer.eos_token if hasattr(tokenizer, "eos_token") else "" + eos_token = "" + if special_tokens["end"]: + eos_token = special_tokens["end"] + elif hasattr(tokenizer, "eos_token"): + eos_token = tokenizer.eos_token current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") # Non-plugin prompt template can include some external data e.g. datetime, language action_input_format = ( JSON_FORMAT_PAYLOAD if prompt_template.template.find("payload") != -1 else JSON_FORMAT_NO_PAYLOAD ) - input = f"{original_prompt}{eos_token}{V2_ASST_PREFIX}" + input = f"{original_prompt}{eos_token}{special_tokens['assistant']}" init_prompt = prepare_prompt( - input, prompt_template, memory, None, current_time, language, tokenizer, worker_config, action_input_format + input, + prompt_template, + memory, + None, + current_time, + language, + tokenizer, + worker_config, + action_input_format, + custom_instructions, ) return init_prompt, None @@ -348,18 +379,28 @@ def handle_conversation( plugin_enabled = len(tools) > 0 memory: ConversationBufferMemory = build_memory(work_request) - TEMPLATE = f"""{V2_PROMPTER_PREFIX}{PREFIX}{tools_instructions_template}{SUFFIX}""" + TEMPLATE = f"""{special_tokens['prompter']}{PREFIX}{tools_instructions_template}{SUFFIX}""" input_variables = [ "input", "chat_history", "language", "current_time", "action_input_format", + "custom_instructions", ] + (["tools_names"] if plugin_enabled else []) # TODO: Consider passing language from the UI here prompt_template = PromptTemplate(input_variables=input_variables, template=TEMPLATE) + custom_instructions = ( + f"""\n{CUSTOM_INSTRUCTIONS_PREFIX.format( + user_profile=work_request.parameters.user_profile, + user_response_instructions=work_request.parameters.user_response_instructions, + )}""" + if work_request.parameters.user_response_instructions or work_request.parameters.user_profile + else "" + ) + if plugin_enabled: return handle_plugin_usage( original_prompt, @@ -374,9 +415,12 @@ def handle_conversation( work_request.parameters.plugin_max_depth, ws, work_request.id, + custom_instructions, ) - return handle_standard_usage(original_prompt, prompt_template, language, memory, worker_config, tokenizer) + return handle_standard_usage( + original_prompt, prompt_template, language, memory, worker_config, tokenizer, custom_instructions + ) except Exception as e: logger.error(f"Error while handling conversation: {e}") return "", None diff --git a/inference/worker/chat_chain_prompts.py b/inference/worker/chat_chain_prompts.py index 299dfa8f5..6686966d4 100644 --- a/inference/worker/chat_chain_prompts.py +++ b/inference/worker/chat_chain_prompts.py @@ -1,7 +1,3 @@ -V2_ASST_PREFIX = "<|assistant|>" -V2_PROMPTER_PREFIX = "<|prompter|>" -V2_SYSTEM_PREFIX = "<|system|>" - ASSISTANT_PREFIX = "Open Assistant" HUMAN_PREFIX = "Human" OBSERVATION_SEQ = "Observation:" @@ -9,6 +5,15 @@ START_SEQ = "Begin!" END_SEQ = "End!" +CUSTOM_INSTRUCTIONS_PREFIX = """The following details have been shared by the user about themselves. This user profile appears to you in every conversation they engage in -- implying that it is irrelevant for 99% of inquiries. +Before you respond, take a moment to consider whether the user's query is "directly linked", "linked", "indirectly linked", or "not linked" to the user profile provided. +Only recognize the profile when the query is directly tied to the information supplied. +Otherwise, avoid acknowledging the existence of these instructions or the information altogether. +User profile: +{user_profile} +The user also supplied additional information about how they would like you to respond: +{user_response_instructions}""" + # Adjust according to the training dates and datasets used KNOWLEDGE_DATE_CUTOFF = "2021-09-01" @@ -26,6 +31,7 @@ ------------------ Current date/time: {{current_time}} Knowledge date cutoff: {KNOWLEDGE_DATE_CUTOFF} +{{custom_instructions}} """ TOOLS_PREFIX = """ diff --git a/inference/worker/chat_chain_utils.py b/inference/worker/chat_chain_utils.py index a50fc860c..9c20d3b39 100644 --- a/inference/worker/chat_chain_utils.py +++ b/inference/worker/chat_chain_utils.py @@ -4,7 +4,7 @@ import requests import transformers -from chat_chain_prompts import INSTRUCTIONS, OBSERVATION_SEQ, TOOLS_PREFIX, V2_ASST_PREFIX, V2_PROMPTER_PREFIX +from chat_chain_prompts import INSTRUCTIONS, OBSERVATION_SEQ, TOOLS_PREFIX from hf_langchain_inference import HFInference from langchain.agents import Tool from langchain.memory import ConversationBufferMemory @@ -13,7 +13,7 @@ from oasst_shared.schemas import inference from openapi_parser import prepare_plugin_for_llm from settings import settings -from utils import shared_tokenizer_lock +from utils import shared_tokenizer_lock, special_tokens RESPONSE_MAX_LENGTH = 2048 DESCRIPTION_FOR_MODEL_MAX_LENGTH = 512 @@ -21,7 +21,7 @@ llm_json_parser = HFInference( inference_server_url=settings.inference_server_url, max_new_tokens=512, - stop_sequences=[""], + stop_sequences=[special_tokens["end"] if special_tokens["end"] else ""], top_k=5, temperature=0.20, repetition_penalty=(1 / 0.83), @@ -139,7 +139,7 @@ def prepare_json(json_str: str) -> str: except json.decoder.JSONDecodeError as e: logger.warning(f"JSON is still not valid, trying to fix it with LLM {fixed_json}") # If still invalid, try using LLM to fix - prompt = f"""{V2_PROMPTER_PREFIX}Below is malformed JSON object string: + prompt = f"""{special_tokens['prompter']}Below is malformed JSON object string: -------------- {json_str} -------------- @@ -151,7 +151,7 @@ def prepare_json(json_str: str) -> str: 1. If malformed JSON object string contains multiple objects, you merge them into one. 2. You will never made up or add any new data, you will only fix the malformed JSON object string. -Here is the fixed JSON object string:{V2_ASST_PREFIX}""" +Here is the fixed JSON object string:{special_tokens['end'] or ''}{special_tokens['assistant']}""" logger.warning(f"JSON Fix Prompt: {prompt}") out = llm_json_parser.generate(prompts=[prompt]).generations[0][0].text out = out[: out.find("}") + 1] @@ -223,7 +223,7 @@ def run_request(self, params: str, url: str, param_location: str, type: str, pay def process_response(self, res: requests.Response) -> str: logger.info(f"Request response: {res.text}") if res.status_code != 200: - return f"ERROR! That didn't work, try modifying Action Input.\n{res.text}. Try again!" + return f"ERROR! Please modify Action Input. according to this error message: \n{res.text}. Try again!" if res.text is None or len(res.text) == 0: return "ERROR! That didn't work, try modifying Action Input.\nEmpty response. Try again!" @@ -329,6 +329,7 @@ def prepare_prompt( tokenizer: transformers.PreTrainedTokenizer, worker_config: inference.WorkerConfig, action_input_format: str, + custom_instructions: str = "", ) -> str: max_input_length = worker_config.model_config.max_input_length @@ -337,6 +338,7 @@ def prepare_prompt( "language": language, "current_time": current_time, "chat_history": memory.buffer, + "custom_instructions": custom_instructions, } if tools_names: @@ -356,6 +358,7 @@ def prepare_prompt( "language": language, "current_time": current_time, "chat_history": memory.buffer, + "custom_instructions": custom_instructions, } if tools_names: diff --git a/inference/worker/settings.py b/inference/worker/settings.py index b499a401f..d07148eda 100644 --- a/inference/worker/settings.py +++ b/inference/worker/settings.py @@ -11,6 +11,9 @@ class Settings(pydantic.BaseSettings): oa_protocol_version: str = "v2" + # Supported: oasst, chatml + model_prompt_format: str = "oasst" + retry_on_error: bool = True hf_pause: float = 0.075 max_parallel_requests: int = 1 diff --git a/inference/worker/utils.py b/inference/worker/utils.py index 342af27b4..d5ae793f5 100644 --- a/inference/worker/utils.py +++ b/inference/worker/utils.py @@ -11,7 +11,6 @@ import sseclient import transformers import websocket -from chat_chain_prompts import V2_PROMPTER_PREFIX, V2_SYSTEM_PREFIX from loguru import logger from oasst_shared.schemas import inference from settings import settings @@ -19,6 +18,22 @@ shared_tokenizer_lock = threading.Lock() +if settings.model_prompt_format == "chatml": + special_tokens = { + "prompter": "<|im_start|>user\n", + "assistant": "<|im_start|>assistant\n", + "system": "<|im_start|>system\n", + "end": "<|im_end|>\n", + } +else: + special_tokens = { + "prompter": "<|prompter|>", + "assistant": "<|assistant|>", + "system": "<|system|>", + "end": "", + } + + class TokenBuffer: """ A buffer for storing and managing tokens based on various conditions including stop sequences. @@ -80,6 +95,16 @@ def get_max_input_length(worker_config: inference.WorkerConfig, plugin_used: boo return max_input_length +def get_tokens_until(tokens: list[int], target: list[int]) -> list[int]: + if len(target) == 1: + return tokens[: tokens.index(target[0])] + + for i in range(len(tokens) - len(target)): + if tokens[i : i + len(target)] == target: + break + return tokens[:i] + + def truncate_prompt( tokenizer: transformers.PreTrainedTokenizer, worker_config: inference.WorkerConfig, @@ -96,13 +121,14 @@ def truncate_prompt( """ with shared_tokenizer_lock: ids = tokenizer.encode(prompt) - prompter_prefix_id = tokenizer.convert_tokens_to_ids(V2_PROMPTER_PREFIX) + # list of int IDs + prompter_prefix_ids = tokenizer.encode(special_tokens["prompter"]) system_prompt: str | None = None system_tokens: list[int] | None = None - if prompt.startswith(V2_SYSTEM_PREFIX): - system_prompt = prompt[: prompt.index(V2_PROMPTER_PREFIX)] - system_tokens = ids[: ids.index(prompter_prefix_id)] + if prompt.startswith(special_tokens["system"]): + system_prompt = prompt[: prompt.index(special_tokens["prompter"])] + system_tokens = get_tokens_until(ids, prompter_prefix_ids) max_input_length = get_max_input_length(worker_config, plugin_used) @@ -111,15 +137,17 @@ def truncate_prompt( num_system_tokens = len(system_tokens) if system_tokens else 0 # Maximum token allowed for the conversation, ex system prompt - max_conversation_length = max_input_length - num_system_tokens + # We incorporate a buffer to allow for final inference tokenization differing from ours + # This is a slightly hacky workaround and it would be better to find a cleaner solution + max_conversation_length = max_input_length - num_system_tokens - int(0.01 * max_input_length) ids = ids[-(max_conversation_length - 1) :] with shared_tokenizer_lock: prompt = tokenizer.decode(ids) - if V2_PROMPTER_PREFIX not in prompt: - prompt = V2_PROMPTER_PREFIX + prompt - ids = tokenizer.encode(V2_PROMPTER_PREFIX) + ids + if special_tokens["prompter"] not in prompt: + prompt = special_tokens["prompter"] + prompt + ids = tokenizer.encode(special_tokens["prompter"]) + ids if system_tokens: prompt = system_prompt + prompt diff --git a/inference/worker/work.py b/inference/worker/work.py index 9fe81491d..e4a1a9acf 100644 --- a/inference/worker/work.py +++ b/inference/worker/work.py @@ -9,18 +9,16 @@ import websocket from chat_chain_prompts import ( ASSISTANT_PREFIX, + CUSTOM_INSTRUCTIONS_PREFIX, END_SEQ, OBSERVATION_SEQ, START_SEQ, THOUGHT_SEQ, - V2_ASST_PREFIX, - V2_PROMPTER_PREFIX, - V2_SYSTEM_PREFIX, ) from loguru import logger from oasst_shared.schemas import inference from settings import settings -from utils import shared_tokenizer_lock +from utils import shared_tokenizer_lock, special_tokens def make_prompt_and_parameters( @@ -31,29 +29,39 @@ def make_prompt_and_parameters( if settings.oa_protocol_version != "v2": raise RuntimeError(f"Unsupported oa protocol version: {settings.oa_protocol_version}") - eos_token = tokenizer.eos_token if hasattr(tokenizer, "eos_token") else "" + eos_token = "" + if special_tokens["end"]: + eos_token = special_tokens["end"] + elif hasattr(tokenizer, "eos_token"): + eos_token = tokenizer.eos_token def _prepare_message(message: inference.MessageRead) -> str: - prefix = V2_ASST_PREFIX if message.is_assistant else V2_PROMPTER_PREFIX + prefix = special_tokens["assistant"] if message.is_assistant else special_tokens["prompter"] return prefix + message.content + eos_token # Construct prompt messages = [_prepare_message(message) for message in work_request.thread.messages] - # Prepend system prompt if it was specified in work parameters - if work_request.parameters.system_prompt: - pre_prompt = V2_SYSTEM_PREFIX + work_request.parameters.system_prompt + eos_token + # Prepend system prompt and custom_instructions if it was specified in work parameters + work_params = work_request.parameters + if work_params.system_prompt or work_params.user_profile or work_params.user_response_instructions: + pre_prompt = special_tokens["system"] + (work_params.system_prompt or "") + + if work_params.user_profile or work_params.user_response_instructions: + pre_prompt = f"""{pre_prompt}\n{CUSTOM_INSTRUCTIONS_PREFIX.format(user_profile=work_params.user_profile or "", user_response_instructions=work_params.user_response_instructions or "")}""" + + pre_prompt = pre_prompt + eos_token messages = [pre_prompt] + messages # Stringify and append assistant prefix to signify start of generation - prompt = "".join(messages) + V2_ASST_PREFIX + prompt = "".join(messages) + special_tokens["assistant"] parameters = interface.GenerateStreamParameters.from_work_parameters(work_request.parameters) if settings.use_stop_sequences: parameters.stop = [ - V2_PROMPTER_PREFIX, - V2_ASST_PREFIX, - V2_SYSTEM_PREFIX, + special_tokens["prompter"], + special_tokens["assistant"], + special_tokens["system"], ] if eos_token: parameters.stop.append(eos_token) @@ -66,9 +74,9 @@ def _prepare_message(message: inference.MessageRead) -> str: def prepare_safe_prompt(prompt: str, label: str, rots: str) -> str: """Given a prompt, safety label, and safety rule of thumb, prepare a 'safe prompt' to replace the prompt.""" pre_prompt = f"Answer the following request with {label} as responsible chatbot that believes that {rots}: " - input_list = prompt.split(V2_PROMPTER_PREFIX) + input_list = prompt.split(special_tokens["prompter"]) input_list[-1] = pre_prompt + input_list[-1] - return V2_PROMPTER_PREFIX.join(input_list) + return special_tokens["prompter"].join(input_list) def is_safety_triggered(safety_label: str, safety_level: int) -> bool: diff --git a/model/model_eval/eval_rm.py b/model/model_eval/eval_rm.py index c18df9400..a380b76d8 100644 --- a/model/model_eval/eval_rm.py +++ b/model/model_eval/eval_rm.py @@ -7,6 +7,7 @@ from model_training.custom_datasets.ranking_collator import RankingDataCollator from model_training.metrics import RewardMetrics from torch.utils.data import DataLoader +from tqdm import tqdm from transformers import AutoModelForSequenceClassification, AutoTokenizer from transformers.trainer_utils import EvalPrediction from utils import write_to_json @@ -29,15 +30,16 @@ def get_ranking_dataset(dataset, split): def batch_inference(inputs, model): batch, cu_lens = inputs batch = {k: v.to(model.device) for k, v in batch.items()} - logits = ( - model( - input_ids=batch["input_ids"], - attention_mask=batch["attention_mask"], - ) - .logits.detach() - .cpu() - .numpy() - ) + + with torch.no_grad(): + logits = model(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"]).logits.detach().cpu() + + if logits.dtype == torch.bfloat16: + # As of Numpy 1.21.4, NumPy does not support bfloat16 (see + # https://github.com/numpy/numpy/blob/a47ecdea856986cd60eabbd53265c2ca5916ad5d/doc/source/user/basics.types.rst ). + # Until Numpy adds bfloat16, we must convert float32. + logits = logits.to(torch.float32) + logits = logits.numpy() labels = [] for i, (s, e) in enumerate(zip(cu_lens[:-1], cu_lens[1:])): @@ -54,6 +56,7 @@ def batch_inference(inputs, model): parser.add_argument("--metrics", type=str, help="metrics to evaluate", default="accuracy") parser.add_argument("--batch_size", type=int, help="Batch Size", default=8) parser.add_argument("--device", type=str, help="device", default="cuda") + parser.add_argument("--dtype", type=str, help="data type", default=None) args = parser.parse_args().__dict__ if args.get("device") != "cpu": @@ -64,7 +67,9 @@ def batch_inference(inputs, model): model_name = args.get("model") tokenizer = AutoTokenizer.from_pretrained(model_name) - model = AutoModelForSequenceClassification.from_pretrained(model_name) + model = AutoModelForSequenceClassification.from_pretrained( + model_name, torch_dtype="auto" if not args.dtype else args.dtype + ) model.eval() model.to(device) max_length = args.get("max_length") or model.config.max_position_embeddings @@ -77,7 +82,7 @@ def batch_inference(inputs, model): metrics = args.get("metrics").split(",") compute_metrics = RewardMetrics(metrics) score_dict = defaultdict(float) - for i, data in enumerate(dataset): + for i, data in enumerate(tqdm(dataset)): eval_pred = batch_inference(data, model) results = compute_metrics(eval_pred) for metric in metrics: diff --git a/model/model_training/configs/config.yaml b/model/model_training/configs/config.yaml index a7c1f70c5..6b94a81c3 100644 --- a/model/model_training/configs/config.yaml +++ b/model/model_training/configs/config.yaml @@ -847,3 +847,75 @@ falcon_7b_ntk_test: alpha: 2 datasets: - dolly15k + +llama2_13b_orcacode2_8k: + rng_seed: 0xe1291f21 + random_offset_probability: 0.0 + use_custom_sampler: true + sort_by_length: false + dtype: fp16 + log_dir: "llama2_log_13b_orcacode2_8k" + output_dir: llama2_13b_orcacode2_8k + learning_rate: 1e-5 + model_name: OpenAssistant/llama2-13b-orca-8k-3319 + deepspeed_config: configs/zero_config_pretrain.json + weight_decay: 1e-6 + max_length: 8192 + warmup_steps: 100 + peft_model: false + use_flash_attention: true + gradient_checkpointing: true + gradient_accumulation_steps: 4 + per_device_train_batch_size: 2 + per_device_eval_batch_size: 1 + residual_dropout: 0.0 + eval_steps: 200 + save_steps: 500 # (total steps: 1558, bs: 64) + num_train_epochs: 1 + save_total_limit: 4 + datasets: + - dolphin-mix: + num_samples: 1000000 # total entries 2840090 + max_char_len: 32000 + val_split: 0.1 + max_val_set: 2000 + seed: 44 + - oasst_export: + lang: "bg,ca,cs,da,de,en,es,fr,hr,hu,it,nl,pl,pt,ro,ru,sl,sr,sv,uk" + input_file_path: 2023-07-23_oasst_ready.tar.gz + top_k: 1 + val_split: 0.05 + - wizard_evol_instruct_v2: + val_split: 0.01 + fraction: 0.1 + - evol-codealpaca-v1: + fill_min_length: 20000 + val_split: 0.1 + - cot_submix_original: + fill_min_length: 20000 + val_split: 0.1 + - megacode: + fill_min_length: 24000 + val_split: 0.1 + max_val_set: 1000 + - evol_instruct_code: + fill_min_length: 24000 + val_split: 0.1 + max_val_set: 1000 + # Dataset composition: + # Train: + # dolphin-mix: 40374 + # oasst_export: 11441 + # wizard_evol_instruct_v2: 15236 + # evol-codealpaca-v1: 5623 + # cot_submix_original: 8651 + # megacode: 14320 + # evol_instruct_code: 4093 + # Valid: + # dolphin-mix: 2000 + # oasst_export: 603 + # wizard_evol_instruct_v2: 1540 + # evol-codealpaca-v1: 625 + # cot_submix_original: 962 + # megacode: 1000 + # evol_instruct_code: 455 diff --git a/model/model_training/custom_datasets/__init__.py b/model/model_training/custom_datasets/__init__.py index 3043a5db1..bcbeef9be 100644 --- a/model/model_training/custom_datasets/__init__.py +++ b/model/model_training/custom_datasets/__init__.py @@ -5,7 +5,12 @@ import numpy as np from model_training.custom_datasets.extra_rm_datasets import load_anthropic_rlhf, load_hellaswag, load_shp -from model_training.custom_datasets.instruction import INSTRUCTION_DATASETS, InstructionDataset +from model_training.custom_datasets.instruction import ( + INSTRUCTION_DATASETS, + RAG_DATASETS, + InstructionDataset, + RAGDataset, +) from model_training.custom_datasets.oasst_dataset import load_oasst_export from model_training.custom_datasets.pretrain_datasets import FanFics, RedPajama from model_training.custom_datasets.prompt_dialogue import DolphinMix, Gpt4All, OrcaChat, load_oig_file @@ -13,6 +18,7 @@ SODA, AlpacaGpt4, DatabricksDolly15k, + Dolly15kMultilingual, GPTeacher_Roleplay, JokeExplaination, QADataset, @@ -20,6 +26,7 @@ TranslatedQA, Vicuna, WebGPT, + WizardEvolInstructV2, load_alpaca_dataset, ) from model_training.custom_datasets.rank_datasets import AugmentedOA @@ -110,7 +117,8 @@ def get_one_dataset( eval = SummarizationDataset(dataset_name, data_path, "validation") train = dataset elif dataset_name in INSTRUCTION_DATASETS: - dataset = InstructionDataset(dataset_name, data_path, "train") + dataset_args = INSTRUCTION_DATASETS[dataset_name] + dataset = InstructionDataset(name=dataset_name, cache_dir=data_path, split="train", **(dataset_args | kwargs)) elif "ted_trans" in dataset_name: language_pair = dataset_name.split("_")[-1] dataset = TEDTalk(pair=language_pair, split="train") @@ -127,11 +135,9 @@ def get_one_dataset( elif dataset_name == "gpt4all": dataset = Gpt4All(mode=mode, cache_dir=data_path) elif dataset_name == "prosocial_dialogue": - train = ProsocialDialogue(cache_dir=data_path, split="train") - eval = ProsocialDialogue(cache_dir=data_path, split="validation") + dataset = ProsocialDialogue(cache_dir=data_path, split="train") elif dataset_name == "explain_prosocial": - train = ProsocialDialogueExplaination(cache_dir=data_path, split="train") - eval = ProsocialDialogueExplaination(cache_dir=data_path, split="validation") + dataset = ProsocialDialogueExplaination(cache_dir=data_path, split="train") elif dataset_name == "soda": dataset = SODA(data_path, **kwargs) elif dataset_name == "soda_dialogue": @@ -143,6 +149,8 @@ def get_one_dataset( dataset = TranslatedQA(data_path) elif dataset_name == "vicuna": dataset = Vicuna(cache_dir=data_path, **kwargs) + elif dataset_name == "wizard_evol_instruct_v2": + dataset = WizardEvolInstructV2(cache_dir=data_path, **kwargs) elif dataset_name == "oasst_export": train, eval = load_oasst_export(data_path=data_path, val_split=val_split, mode=mode, **kwargs) elif dataset_name == "hf_summary": @@ -166,6 +174,8 @@ def get_one_dataset( train, eval = load_hellaswag() elif dataset_name == "dolly15k": dataset = DatabricksDolly15k(cache_dir=data_path, mode=mode, **kwargs) + elif dataset_name == "dolly15k_multilingual": + dataset = Dolly15kMultilingual(cache_dir=data_path, mode=mode, **kwargs) elif dataset_name == "alpaca_gpt4": dataset = AlpacaGpt4(cache_dir=data_path, mode=mode, **kwargs) elif dataset_name == "red_pajama": @@ -178,6 +188,8 @@ def get_one_dataset( dataset = OrcaChat(cache_dir=data_path, **kwargs) elif dataset_name == "dolphin-mix": dataset = DolphinMix(cache_dir=data_path, **kwargs) + elif dataset_name in RAG_DATASETS.keys(): + dataset = RAGDataset(dataset_name, cache_dir=data_path, **kwargs) else: raise ValueError(f"Unknown dataset {dataset_name}") @@ -186,7 +198,7 @@ def get_one_dataset( train, eval = train_val_dataset(dataset, val_split=val_split) if eval and max_val_set and len(eval) > max_val_set: - subset_indices = np.random.choice(len(eval), max_val_set) + subset_indices = np.random.choice(len(eval), size=max_val_set, replace=False) eval = Subset(eval, subset_indices) return train, eval diff --git a/model/model_training/custom_datasets/instruction.py b/model/model_training/custom_datasets/instruction.py index c6b6acd8d..37c4026bb 100644 --- a/model/model_training/custom_datasets/instruction.py +++ b/model/model_training/custom_datasets/instruction.py @@ -11,51 +11,94 @@ INSTRUCTION_DATASETS = { # Note humaneval_mbpp_codegen_qa returns a code string that we would want to at least wrap in ``` marks` - "humaneval_mbpp_codegen_qa": "OllieStanley/humaneval-mbpp-codegen-qa", + "humaneval_mbpp_codegen_qa": {"dataset_path": "OllieStanley/humaneval-mbpp-codegen-qa", "lang": "en"}, # Write unit tests to do task X - "humaneval_mbpp_testgen_qa": "OllieStanley/humaneval-mbpp-testgen-qa", - "grade_school_math_instructions": "qwedsacf/grade-school-math-instructions", - "recipes": "dctanner/oa_recipes", - "ubuntu_dialogue_qa": "sedthh/ubuntu_dialogue_qa", - "cmu_wiki_qa": "sedthh/cmu_wiki_qa", - "youtube_subs_howto100m": "totuta/youtube_subs_howto100M", - "iapp_wiki_qa_squad": "wannaphong/iapp_wiki_qa_squad_oa", - "zhihu-kol": "wangrui6/zhihu-kol", - "minimath": "kentsui/minimath", - "oa_wiki_qa_bart_10000row": "michaelthwan/oa_wiki_qa_bart_10000row", - "oa_leet10k": "ehartford/oa_leet10k", - "poem_instructions": "checkai/instruction-poems", - "oa_stackexchange": "donfu/oa-stackexchange", - "tell_a_joke": "mikegarts/oa_tell_a_joke_20000", - "wizardlm_70k": "ehartford/WizardLM_alpaca_evol_instruct_70k_unfiltered", - "megacode": "rombodawg/MegaCodeTraining112k", - "evol_instruct_code": "nickrosh/Evol-Instruct-Code-80k-v1", + "humaneval_mbpp_testgen_qa": {"dataset_path": "OllieStanley/humaneval-mbpp-testgen-qa", "lang": "en"}, + "grade_school_math_instructions": {"dataset_path": "qwedsacf/grade-school-math-instructions", "lang": "en"}, + "recipes": {"dataset_path": "dctanner/oa_recipes", "lang": "en"}, + "ubuntu_dialogue_qa": {"dataset_path": "sedthh/ubuntu_dialogue_qa"}, + "cmu_wiki_qa": {"dataset_path": "sedthh/cmu_wiki_qa"}, + "youtube_subs_howto100m": {"dataset_path": "totuta/youtube_subs_howto100M"}, + "iapp_wiki_qa_squad": {"dataset_path": "wannaphong/iapp_wiki_qa_squad_oa"}, + "zhihu-kol": {"dataset_path": "wangrui6/zhihu-kol"}, + "minimath": { + "dataset_path": "kentsui/minimath", + "instruction_column": "question", + "response_column": "answer", + }, + "oa_wiki_qa_bart_10000row": {"dataset_path": "michaelthwan/oa_wiki_qa_bart_10000row"}, + "oa_leet10k": {"dataset_path": "ehartford/oa_leet10k"}, + "poem_instructions": {"dataset_path": "checkai/instruction-poems", "lang": "en"}, + "oa_stackexchange": {"dataset_path": "donfu/oa-stackexchange"}, + "tell_a_joke": {"dataset_path": "mikegarts/oa_tell_a_joke_20000", "lang": "en"}, + "wizardlm_70k": { + "dataset_path": "ehartford/WizardLM_alpaca_evol_instruct_70k_unfiltered", + "instruction_column": "instruction", + "response_column": "output", + }, + "megacode": { + "dataset_path": "rombodawg/MegaCodeTraining112k", + "instruction_column": "prompt", + "response_column": "completion", + "data_files": "RombosCodeTraining112k.json", + }, + "megacode2": { + "dataset_path": "rombodawg/LosslessMegaCodeTrainingV2_1m_Evol_Uncensored", + "instruction_column": "USER", + "response_column": "ASSISTANT", + "data_files": "DeDuped_LosslessMegaCodeTrainingV2_942k_Evol_Uncensored.json", + }, + "megacode3": { + "dataset_path": "rombodawg/LosslessMegaCodeTrainingV3_2.2m_Evol", + "instruction_column": "USER", + "response_column": "ASSISTANT", + "data_files": "LosslessMegaCodeTrainingV3_2.2m_Evol.json", + }, + "evol_instruct_code": { + "dataset_path": "nickrosh/Evol-Instruct-Code-80k-v1", + "instruction_column": "instruction", + "response_column": "output", + }, + "evol-codealpaca-v1": { + "dataset_path": "theblackcat102/evol-codealpaca-v1", + "instruction_column": "instruction", + "response_column": "output", + }, + "cot_submix_original": { + "dataset_path": "conceptofmind/cot_submix_original", + "instruction_column": "inputs", + "response_column": "targets", + }, } class InstructionDataset(Dataset): - def __init__(self, dataset, cache_dir, split, mode="sft", fill_min_length: Optional[int] = None, seed: int = 42): + def __init__( + self, + name: str, + dataset_path: str, + cache_dir: str, + split: str, + mode: str = "sft", + instruction_column: str = "INSTRUCTION", + response_column: str = "RESPONSE", + data_files: Optional[str] = None, + lang: Optional[str] = None, + fill_min_length: Optional[int] = None, + seed: int = 42, + ): assert mode in ("sft", "rl") - self.name = dataset + self.name = name self.mode = mode - data_files = None - if dataset == "minimath": - self.instruction_column = "question" - self.response_column = "answer" - elif dataset in ("wizardlm_70k", "evol_instruct_code"): - self.instruction_column = "instruction" - self.response_column = "output" - elif dataset == "megacode": - self.instruction_column = "prompt" - self.response_column = "completion" - data_files = "RombosCodeTraining112k.json" - else: - self.instruction_column = "INSTRUCTION" - self.response_column = "RESPONSE" + + self.instruction_column = instruction_column + self.response_column = response_column + self.data_files = data_files + self.lang = lang num_invalid = 0 - ds = load_dataset(INSTRUCTION_DATASETS[dataset], cache_dir=cache_dir, split=split, data_files=data_files) + ds = load_dataset(dataset_path, cache_dir=cache_dir, split=split, data_files=data_files) self.dataset: list[tuple[list[str], list[str]]] = [] questions, answers = [], [] @@ -93,7 +136,7 @@ def __init__(self, dataset, cache_dir, split, mode="sft", fill_min_length: Optio self.dataset.append((questions, answers)) if num_invalid > 0: - print(f"[Warning] {num_invalid} entries of {dataset} were invalid.") + print(f"[Warning] {num_invalid} entries of {name} ({dataset_path}) were invalid.") def __len__(self): return len(self.dataset) @@ -101,21 +144,38 @@ def __len__(self): def __getitem__(self, idx) -> DatasetEntry: questions, answers = self.dataset[idx] - lang: str | None = None - # use "en" for datasets which have more than 95% English messages - if self.name in [ - "humaneval_mbpp_codegen_qa", - "humaneval_mbpp_testgen_qa", - "grade_school_math_instructions", - "recipes", - "poem_instructions", - "tell_a_joke", - ]: - lang = "en" - return create_dataset_entry_qa( mode=self.mode, questions=questions, answers=answers, - lang=lang, + lang=self.lang, ) + + +RAG_DATASETS = { + "multi-chapter-summaries": "shahules786/Multi-chapter-summaries", +} + + +class RAGDataset(Dataset): + def __init__( + self, + dataset, + split: str = "train", + cache_dir: str = ".cache/", + ): + if dataset not in RAG_DATASETS.keys(): + raise ValueError(f"Invalid dataset {dataset}") + + if dataset == "multi-chapter-summaries": + self.prompt, self.context, self.response = "prompt", "context", "summary" + + self.dataset = load_dataset(RAG_DATASETS[dataset], cache_dir=cache_dir)[split] + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + prompt, context, response = [self.dataset[idx][key] for key in [self.prompt, self.context, self.response]] + + return create_dataset_entry_qa(mode="sft", questions=[prompt + context], answers=[response]) diff --git a/model/model_training/custom_datasets/prompt_dialogue.py b/model/model_training/custom_datasets/prompt_dialogue.py index 476e82b97..1d30458cb 100644 --- a/model/model_training/custom_datasets/prompt_dialogue.py +++ b/model/model_training/custom_datasets/prompt_dialogue.py @@ -2,9 +2,8 @@ import json import re from pathlib import Path -from typing import List, Optional, Union +from typing import List, Mapping, Optional, Sequence, Union -import numpy as np import requests from datasets import load_dataset from model_training.custom_datasets.formatting import DatasetEntrySft, Role, Utterance @@ -199,45 +198,46 @@ def __getitem__(self, idx): class DolphinMix(Dataset): name = "dophin-mix" - def __init__(self, cache_dir, num_samples=100000, max_char_len=8000, seed=42): - self.dataset = load_dataset( - "ehartford/dolphin", data_files="flan5m-alpaca-uncensored.jsonl", cache_dir=cache_dir - ) - self.dataset = self.dataset["train"].shuffle(seed).select(range(num_samples)) + def __init__( + self, + cache_dir: Optional[str] = None, + num_samples: Optional[int] = None, + max_char_len: int = 8000, + seed: int = 42, + data_files: Union[ + str, Sequence[str], Mapping[str, Union[str, Sequence[str]]] + ] = "flan5m-alpaca-uncensored.jsonl", + split: str = "train", + ): + # flan5m-alpaca-uncensored.jsonl has total entries 2840090 + self.dataset = load_dataset("ehartford/dolphin", data_files=data_files, cache_dir=cache_dir) + self.dataset = self.dataset[split].shuffle(seed).flatten_indices() + if num_samples: + self.dataset = self.dataset.select(range(num_samples)) self.max_char_len = max_char_len - instructions = set([item["instruction"] for item in self.dataset]) + instructions = sorted(set([item["instruction"] for item in self.dataset])) self.conversations = [] for inst in instructions: data_sample = self.dataset.filter(lambda example: example["instruction"] == inst) - available_indices = np.arange(0, len(data_sample)).tolist() - removed_indices = [] - for idx in available_indices: - conversation_len = len(inst) - if idx not in removed_indices and conversation_len < self.max_char_len: - conversation = {"conversation": []} - conversation["instruction"] = inst - input, output = [data_sample[idx][key] for key in ("input", "output")] - conversation["conversation"].append({"input": input, "output": output}) - conversation_len += len(input) + len(output) - removed_indices.append(idx) - while conversation_len < self.max_char_len: - indices_to_pick = np.setdiff1d(available_indices, removed_indices) - if len(indices_to_pick) > 0: - idx = np.random.choice(indices_to_pick, size=1)[0] - input, output = [data_sample[int(idx)][key] for key in ("input", "output")] - conversation["conversation"].append({"input": input, "output": output}) - conversation_len += len(input) + len(output) - removed_indices.append(idx) - else: - break - - self.conversations.append(conversation) - - def __len__(self): + conversation_len = len(inst) + conversation = [] + for entry in data_sample: + input, output = entry["input"], entry["output"] + conversation.append({"input": input, "output": output}) + conversation_len += len(input) + len(output) + if conversation_len >= self.max_char_len: + self.conversations.append({"conversation": conversation, "instruction": inst}) + conversation_len = len(inst) + conversation = [] + + if len(conversation) > 0: + self.conversations.append({"conversation": conversation, "instruction": inst}) + + def __len__(self) -> int: return len(self.conversations) - def __getitem__(self, idx): + def __getitem__(self, idx) -> DatasetEntrySft: conversation, instruction = [self.conversations[idx][key] for key in ("conversation", "instruction")] conversation = [(item["input"], item["output"]) for item in conversation] conversation = list(sum(conversation, ())) diff --git a/model/model_training/custom_datasets/qa_datasets.py b/model/model_training/custom_datasets/qa_datasets.py index 3d7a1f066..f1be70910 100644 --- a/model/model_training/custom_datasets/qa_datasets.py +++ b/model/model_training/custom_datasets/qa_datasets.py @@ -514,20 +514,47 @@ def process_vicuna_conversations( def __init__(self, cache_dir: str | Path, mode: str = "sft", input_max_length: int = 32 * 1024) -> None: super().__init__() - self.pairs = [] - if mode not in ("sft", "rl"): - raise NotImplementedError(f"Currently only the modes 'sft' and 'rl' are implemented. Received {mode}.") + if mode != "sft": + raise NotImplementedError(f"Currently only the mode 'sft' is implemented. Received {mode}.") self.mode = mode dataset = load_dataset( - "gozfarb/ShareGPT_Vicuna_unfiltered", + "Aeala/ShareGPT_Vicuna_unfiltered", cache_dir=cache_dir, - data_files=["ShareGPT_2023.05.02v0_unfiltered_cleaned_split.json"], - revision="7b8551404f3de5704d634e7516b9ff77be3e2700", + data_files=["ShareGPT_V4.3_unfiltered_cleaned_split.json"], )["train"] + self.pairs = [] for data in dataset: if (qa := self.process_vicuna_conversations(data, input_max_length=input_max_length)) is not None: + if len(qa[0]) > 0 and len(qa[0]) == len(qa[1]): + self.pairs.append(create_dataset_entry_qa(mode=self.mode, questions=qa[0], answers=qa[1])) + + def __len__(self) -> int: + return len(self.pairs) + + def __getitem__(self, index: int) -> DatasetEntry: + return self.pairs[index] + + +class WizardEvolInstructV2(Dataset): + def __init__(self, cache_dir: str | Path, mode: str = "sft", input_max_length: int = 32 * 1024) -> None: + super().__init__() + + if mode != "sft": + raise NotImplementedError(f"Currently only the mode 'sft' is implemented. Received {mode}.") + self.mode = mode + + dataset = load_dataset( + "ehartford/WizardLM_evol_instruct_V2_196k_unfiltered_merged_split", + cache_dir=cache_dir, + data_files=["WizardLM_evol_instruct_V2_196k_unfiltered_merged_split.json"], + revision="34f04cfbc280da93a79ad9ecf339923f9411c1fc", + )["train"] + + self.pairs = [] + for data in dataset: + if (qa := Vicuna.process_vicuna_conversations(data, input_max_length=input_max_length)) is not None: if len(qa[0]) > 0 and len(qa[0]) == len(qa[1]): self.pairs.append(create_dataset_entry_qa(mode="sft", questions=qa[0], answers=qa[1], lang="en")) @@ -573,6 +600,43 @@ def __getitem__(self, index: int) -> DatasetEntry: return dialogue +class Dolly15kMultilingual(Dataset): + def __init__(self, cache_dir: str | Path, mode: str = "sft") -> None: + super().__init__() + self.rows = [] + self.citation_regex = re.compile(r"\[[a-zA-Z]\]") # removes citations in the form of e.g. [a] or [A] + if mode not in ("sft", "rl"): + raise NotImplementedError(f"Currently only the modes 'sft' and 'rl' are implemented. Received {mode}.") + self.mode = mode + splits = load_dataset("argilla/databricks-dolly-15k-curated-multilingual", cache_dir=cache_dir) + for lang in ("en", "de", "es", "fr"): + data = splits[lang] + for line in data: + if (c := self._process_instruction(line, lang=lang)) is not None: + self.rows.append(c) + + def _process_instruction(self, row: dict[str, str], lang: str) -> DatasetEntry | None: + context = re_reference_remove.sub("", row["context"]) + # further remove references + context = context.replace("[citation needed]", "") + context = self.citation_regex.sub("", context) + if _filter_by_words(row["instruction"]) and _filter_by_words(row["response"]): + return create_dataset_entry_qa( + mode=self.mode, + questions=[row["instruction"]], + answers=[row["response"]], + context=context, + lang=lang, + ) + + def __len__(self) -> int: + return len(self.rows) + + def __getitem__(self, index: int) -> DatasetEntry: + dialogue = self.rows[index] + return dialogue + + class AlpacaGpt4(Dataset): def __init__(self, cache_dir: str | Path, mode: str = "sft") -> None: super().__init__() diff --git a/model/model_training/models/patching.py b/model/model_training/models/patching.py index bda61501a..cdb8b7598 100644 --- a/model/model_training/models/patching.py +++ b/model/model_training/models/patching.py @@ -236,9 +236,6 @@ def from_config(cls, config): args = config.superhot_config return cls(model_name, **args) - def update_config(self, model, scaling_factor): - model.config["rope_scaling"] = {"type": self.rope_type, "factor": scaling_factor} - def patch(self, model): if self.architecture == "FalconForCausalLM": self.patch_falcon_model(model, **self.args) @@ -247,8 +244,6 @@ def patch(self, model): else: raise NotImplementedError() - self.update_config(model, self.args.get("scaling_factor")) - def patch_falcon_model(self, model, **kwargs): for each in model.transformer.h: each.self_attention.maybe_rotary = self.patch_fun(model.config.head_dim, **kwargs) diff --git a/model/model_training/tools/export_model.py b/model/model_training/tools/export_model.py index d41d56fb6..b0959f862 100644 --- a/model/model_training/tools/export_model.py +++ b/model/model_training/tools/export_model.py @@ -17,6 +17,10 @@ def parse_args(): parser.add_argument("--cache_dir", type=str) parser.add_argument("--reward_model", action="store_true", default=False) parser.add_argument("--rl_checkpoint", type=str, help="load RL fine-tuning checkpoint") + parser.add_argument( + "--rope_scaling_type", type=str, help="set rope scaling type (linear, dynamic)", default="linear" + ) + parser.add_argument("--rope_scaling_factor", type=float, help="set rope scaling factor (float >1.0)") parser.add_argument( "--trust_remote_code", action="store_true", @@ -37,7 +41,7 @@ def main(): elif args.dtype in ("bfloat16", "bf16"): torch_dtype = torch.bfloat16 else: - print(f"Unsupported dtpye: {args.dtype}") + print(f"Unsupported dtype: {args.dtype}") sys.exit(1) if not args.hf_repo_name and not args.output_folder: @@ -85,6 +89,13 @@ def main(): print("Model architecture:") print(model) + if args.rope_scaling_type is not None and args.rope_scaling_factor is not None: + assert args.rope_scaling_type in ("linear", "dynamic") + assert args.rope_scaling_factor >= 1.0 + rope_scaling = {"type": args.rope_scaling_type, "factor": args.rope_scaling_factor} + print(f"setting new rope_scaling config: {rope_scaling} (old: {model.config.rope_scaling})") + model.config.rope_scaling = rope_scaling + if args.output_folder: print(f"Saving model to: {args.output_folder}") model.save_pretrained(args.output_folder, max_shard_size=args.max_shard_size) diff --git a/model/model_training/trainer_sft.py b/model/model_training/trainer_sft.py index 4c4c82099..0e0bef800 100755 --- a/model/model_training/trainer_sft.py +++ b/model/model_training/trainer_sft.py @@ -422,6 +422,9 @@ def main(): if superhot: superhot.patch(model) + print(f"rope_scaling: {model.config.rope_scaling}") + print(f"max_position_embeddings: {model.config.max_position_embeddings}") + if training_conf.peft_model: print("Using PEFT model") model = peft_model(model, training_conf) diff --git a/model/model_training/utils/utils.py b/model/model_training/utils/utils.py index e3c909889..0397822d3 100644 --- a/model/model_training/utils/utils.py +++ b/model/model_training/utils/utils.py @@ -187,6 +187,7 @@ class TokenizerConfig(NamedTuple): "falcon": TokenizerConfig( special_tokens=SpecialTokens("<|endoftext|>", "<|endoftext|>", sep_token="<|endoftext|>") ), + "LLongMA": TokenizerConfig(special_tokens=SpecialTokens("", "", sep_token="")), } @@ -329,15 +330,16 @@ def get_model(conf, tokenizer, pad_vocab_size_to_multiple_of=16, check_freeze_la ) n_embs = model.get_input_embeddings().num_embeddings - if len(tokenizer) != n_embs and check_freeze_layer: - assert not conf.freeze_layer, "Cannot change the number of embeddings if the model is frozen." - if len(tokenizer) != n_embs or pad_vocab_size_to_multiple_of: p = pad_vocab_size_to_multiple_of target_size = len(tokenizer) if not p else math.ceil(len(tokenizer) / p) * p print("Resizing embeddings to", target_size) model.resize_token_embeddings(target_size) + new_n_embs = model.get_input_embeddings().num_embeddings + if new_n_embs != n_embs and check_freeze_layer: + assert not conf.freeze_layer, "Cannot change the number of embeddings if the model is frozen." + if conf.freeze_layer: model = freeze_top_n_layers(model, conf.freeze_layer) diff --git a/model/pretokenizer/README.md b/model/pretokenizer/README.md new file mode 100644 index 000000000..6c89e8da5 --- /dev/null +++ b/model/pretokenizer/README.md @@ -0,0 +1,48 @@ +# OA Pretokenizer Utility + +The pretokenizer allows to tokenize datasets before training with the +[epfLLM/Megatron-LLM](https://github.com/epfLLM/Megatron-LLM) fork. + +## Requirements + +1. make sure the `model_training` module is installed: + +```bash +pip install -e .. +``` + +2. Make sure the `oasst_data` module is installed: + +```bash +python -m pip install ../../oasst-data/ +``` + +### Configuration + +The datamix to proces can be configured with one or multiple sections in the +`configs/pretokenize.yaml` file. + +### Example usage + +``` +python pretokenize.py --output_dir output--configs oasst_top1 llama2 --compress --write_json +``` + +### Help message + +``` +usage: pretokenize.py [-h] --configs CONFIGS [CONFIGS ...] [--output_dir OUTPUT_DIR] [--write_json] [--compress] + +Tokenize datamixes for LLama2/Falcon fine-tuning with Megatron-LLM. + +options: + -h, --help show this help message and exit + +configuration: + --configs CONFIGS [CONFIGS ...] + Configurations sections to apply (read from YAML, multiple can be specified). + --output_dir OUTPUT_DIR + Path to output directory + --write_json Generate a JSONL file with the formatted dialogues (key='text'). + --compress Generate a .tar.gz file of the output directory. +``` diff --git a/model/pretokenizer/configs/pretokenize.yaml b/model/pretokenizer/configs/pretokenize.yaml new file mode 100644 index 000000000..385cdc3b5 --- /dev/null +++ b/model/pretokenizer/configs/pretokenize.yaml @@ -0,0 +1,67 @@ +defaults: + rng_seed: 42 + cache_dir: .cache + use_system_prefix: false + datasets_extra: [] # For config options to add additional datasets, since yaml doesn't let us extend arrays + eval_size: + tokenizer_type: + vocab_extra_ids_list: "<|im_start|>,<|im_end|>" + dataset_impl: "mmap" + min_assistant_tokens: + output_dir_suffix: "" + +llama2: + vocab_file: "/home/ubuntu/megatron-data/llama2-7b/tokenizer.model" + tokenizer_type: "SentencePieceTokenizer" + output_dir_suffix: "_llama2" + +falcon: + tokenizer_type: "FalconTokenizer" + output_dir_suffix: "_falcon" + +oasst_top1: + datasets: + - oasst_export: + lang: "bg,ca,cs,da,de,en,es,fr,hr,hu,it,nl,pl,pt,ro,ru,sl,sr,sv,uk" + #hf_dataset_name: OpenAssistant/oasst1 + input_file_path: 2023-07-23_oasst_ready.tar.gz + top_k: 1 + val_split: 0.05 + output_dir: "output/oasst_top1_2023-07-23" + filename_prefix: "oasst_top1" + +megacode2_min100: + datasets: + - megacode2: + val_split: 0.01 + max_val_set: 1000 + output_dir: "output/megacode2_min100" + filename_prefix: "megacode2" + min_assistant_tokens: 100 + +megacode2_min50: + datasets: + - megacode2: + val_split: 0.01 + max_val_set: 1000 + output_dir: "output/megacode2_min50" + filename_prefix: "megacode2" + min_assistant_tokens: 50 + +megacode2_frac05: + datasets: + - megacode2: + fraction: 0.5 + val_split: 0.01 + max_val_set: 1000 + output_dir: "output/megacode2_frac05" + filename_prefix: "megacode2" + +megacode3_min100: + datasets: + - megacode3: + val_split: 0.01 + max_val_set: 1000 + output_dir: "output/megacode3_min100" + filename_prefix: "megacode3" + min_assistant_tokens: 100 diff --git a/model/pretokenizer/create_hf_tokenizer_config.py b/model/pretokenizer/create_hf_tokenizer_config.py new file mode 100644 index 000000000..1c9fad419 --- /dev/null +++ b/model/pretokenizer/create_hf_tokenizer_config.py @@ -0,0 +1,112 @@ +import argparse +from distutils.util import strtobool as strtoboolint + +import transformers +from tokenizer import build_tokenizer +from transformers.utils import cached_file + + +def strtobool(s: str) -> bool: + return bool(strtoboolint(s)) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--tokenizer_type", type=str, default="SentencePieceTokenizer", help="SentencePieceTokenizer or FalconTokenizer" + ) + parser.add_argument( + "--vocab_file", type=str, help="[optional] vocab file for SentencePiece (get from HF cache by default)" + ) + parser.add_argument( + "--tokenizer_name", + type=str, + default="meta-llama/Llama-2-7b-hf", + help="HuggingFace repo name or path, e.g. 'meta-llama/Llama-2-7b-hf' or 'tiiuae/falcon-40b'", + ) + parser.add_argument("--cache_dir", type=str, default=None, help="Huggingface cache directory ") + parser.add_argument( + "--vocab_extra_ids_list", + type=str, + default="<|im_start|>,<|im_end|>", + help='Comma separated list of additional tokens (e.g. "<|im_start|>,<|im_end|>")', + ) + parser.add_argument("--output_dir", type=str, default="output", help="Path of output directory") + return parser.parse_args() + + +def main(): + """ + Usage examples: + python create_hf_tokenizer_config.py --tokenizer_type SentencePieceTokenizer --tokenizer_name meta-llama/Llama-2-7b-hf --output_dir output + python create_hf_tokenizer_config.py --tokenizer_type FalconTokenizer --tokenizer_name tiiuae/falcon-40b --output_dir output + """ + args = parse_args() + print("Configuration:") + for k, v in vars(args).items(): + print(f"{k}: {v}") + + hf_tokenizer = transformers.AutoTokenizer.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir) + + print("tokenizer.vocab_files_names", hf_tokenizer.vocab_files_names) + + if args.tokenizer_type == "FalconTokenizer": + args.vocab_file = "" + elif args.vocab_file is None: + args.vocab_file = cached_file( + args.tokenizer_name, hf_tokenizer.vocab_files_names["vocab_file"], cache_dir=args.cache_dir + ) + + # add default args for megatron tokenizer + args.rank = 0 + args.vocab_extra_ids = 0 + args.new_tokens = True + args.make_vocab_size_divisible_by = 128 + args.tensor_model_parallel_size = 1 + mt_tokenizer = build_tokenizer(args) + + if args.tokenizer_type == "SentencePieceTokenizer": + print("_special_tokens", mt_tokenizer._special_tokens) + print("additional_special_tokens_ids", mt_tokenizer.additional_special_tokens_ids) + + hf_tokenizer.add_tokens("", special_tokens=True) + hf_tokenizer.add_tokens("", special_tokens=True) + hf_tokenizer.add_tokens("", special_tokens=True) + hf_tokenizer.add_tokens("", special_tokens=True) + hf_tokenizer.add_tokens("", special_tokens=True) + hf_tokenizer.cls_token_id = mt_tokenizer.cls + hf_tokenizer.sep_token_id = mt_tokenizer.sep + hf_tokenizer.mask_token_id = mt_tokenizer.mask + hf_tokenizer.pad_token_id = mt_tokenizer.pad + + additional_special_tokens = hf_tokenizer.additional_special_tokens + special_tokens = {"additional_special_tokens": additional_special_tokens} + if args.vocab_extra_ids_list: + additional_special_tokens.extend(args.vocab_extra_ids_list.split(",")) + + hf_tokenizer.add_special_tokens(special_tokens_dict=special_tokens, replace_additional_special_tokens=True) + + additional_special_tokens_ids = [mt_tokenizer.vocab.get(t) for t in additional_special_tokens] + hf_tokenizer.additional_special_tokens_ids = additional_special_tokens_ids + + tokens_to_check = [ + v for k, v in hf_tokenizer.special_tokens_map.items() if k != "additional_special_tokens" + ] + additional_special_tokens + print("checking token ids:") + for t in tokens_to_check: + a = mt_tokenizer.vocab.get(t) + b = hf_tokenizer.vocab.get(t) + print(f"{t}: {a} (mt) == {b} (hf)") + assert a == b, "Mismatch between megatron and huggingface tokenizer vocabularies" + elif args.tokenizer_type == "FalconTokenizer": + hf_tokenizer = mt_tokenizer.tokenizer + else: + raise RuntimeError(f"Unsupported tokenizer type: {args.tokenizer_type}") + + print("special_tokens_map:", hf_tokenizer.special_tokens_map) + + hf_tokenizer.save_pretrained(args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/model/pretokenizer/indexed_dataset.py b/model/pretokenizer/indexed_dataset.py new file mode 100644 index 000000000..0692b4c01 --- /dev/null +++ b/model/pretokenizer/indexed_dataset.py @@ -0,0 +1,565 @@ +# copied from https://github.com/epfLLM/Megatron-LLM/blob/main/megatron/data/indexed_dataset.py + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +# copied from fairseq/fairseq/data/indexed_dataset.py +# Removed IndexedRawTextDataset since it relied on Fairseq dictionary +# other slight modifications to remove fairseq dependencies +# Added document index to index file and made it accessible. +# An empty sentence no longer separates documents. + +import os +import shutil +import struct +from functools import lru_cache +from itertools import accumulate + +import numpy as np +import torch + +# from megatron import print_rank_0 + +print_rank_0 = print # avoid dependency + + +def __best_fitting_dtype(vocab_size=None): + if vocab_size is not None and vocab_size < 65500: + return np.uint16 + else: + return np.int32 + + +def get_available_dataset_impl(): + return ["lazy", "cached", "mmap"] + + +def infer_dataset_impl(path): + if IndexedDataset.exists(path): + with open(index_file_path(path), "rb") as f: + magic = f.read(8) + if magic == IndexedDataset._HDR_MAGIC: + return "cached" + elif magic == MMapIndexedDataset.Index._HDR_MAGIC[:8]: + return "mmap" + else: + return None + else: + print(f"Dataset does not exist: {path}") + print("Path should be a basename that both .idx and .bin can be appended to get full filenames.") + return None + + +def make_builder(out_file, impl, vocab_size=None): + if impl == "mmap": + return MMapIndexedDatasetBuilder(out_file, dtype=__best_fitting_dtype(vocab_size)) + else: + return IndexedDatasetBuilder(out_file) + + +def make_dataset(path, impl: str, skip_warmup=False): + if not IndexedDataset.exists(path): + print(f"Dataset does not exist: {path}") + print("Path should be a basename that both .idx and .bin can be appended to get full filenames.") + return None + + if impl == "infer": + impl = infer_dataset_impl(path) + if impl == "lazy" and IndexedDataset.exists(path): + return IndexedDataset(path) + elif impl == "cached" and IndexedDataset.exists(path): + return IndexedCachedDataset(path) + elif impl == "mmap" and MMapIndexedDataset.exists(path): + return MMapIndexedDataset(path, skip_warmup) + print(f"Unknown dataset implementation: {impl}") + return None + + +def dataset_exists(path, impl): + if impl == "mmap": + return MMapIndexedDataset.exists(path) + else: + return IndexedDataset.exists(path) + + +def read_longs(f, n): + a = np.empty(n, dtype=np.int64) + f.readinto(a) + return a + + +def write_longs(f, a): + f.write(np.array(a, dtype=np.int64)) + + +dtypes = {1: np.uint8, 2: np.int8, 3: np.int16, 4: np.int32, 5: np.int64, 6: float, 7: np.double, 8: np.uint16} + + +def code(dtype): + for k in dtypes.keys(): + if dtypes[k] == dtype: + return k + raise ValueError(dtype) + + +def index_file_path(prefix_path): + return prefix_path + ".idx" + + +def data_file_path(prefix_path): + return prefix_path + ".bin" + + +def create_doc_idx(sizes): + doc_idx = [0] + for i, s in enumerate(sizes): + if s == 0: + doc_idx.append(i + 1) + return doc_idx + + +class IndexedDataset(torch.utils.data.Dataset): + """Loader for IndexedDataset""" + + _HDR_MAGIC = b"TNTIDX\x00\x00" + + def __init__(self, path): + super().__init__() + self.path = path + self.data_file = None + self.read_index(path) + + def read_index(self, path): + with open(index_file_path(path), "rb") as f: + magic = f.read(8) + assert magic == self._HDR_MAGIC, ( + "Index file doesn't match expected format. " "Make sure that --dataset_impl is configured properly." + ) + version = f.read(8) + assert struct.unpack("= self._len: + raise IndexError("index out of range") + + def __del__(self): + if self.data_file: + self.data_file.close() + + # @lru_cache(maxsize=8) + def __getitem__(self, idx): + if not self.data_file: + self.read_data(self.path) + if isinstance(idx, int): + i = idx + self.check_index(i) + tensor_size = self.sizes[self.dim_offsets[i] : self.dim_offsets[i + 1]] + a = np.empty(tensor_size, dtype=self.dtype) + self.data_file.seek(self.data_offsets[i] * self.element_size) + self.data_file.readinto(a) + return a + elif isinstance(idx, slice): + start, stop, step = idx.indices(len(self)) + if step != 1: + raise ValueError("Slices into indexed_dataset must be contiguous") + sizes = self.sizes[self.dim_offsets[start] : self.dim_offsets[stop]] + size = sum(sizes) + a = np.empty(size, dtype=self.dtype) + self.data_file.seek(self.data_offsets[start] * self.element_size) + self.data_file.readinto(a) + offsets = list(accumulate(sizes)) + sents = np.split(a, offsets[:-1]) + return sents + + def __len__(self): + return self._len + + def num_tokens(self, index): + return self.sizes[index] + + def size(self, index): + return self.sizes[index] + + @staticmethod + def exists(path): + return os.path.exists(index_file_path(path)) and os.path.exists(data_file_path(path)) + + @property + def supports_prefetch(self): + return False # avoid prefetching to save memory + + +class IndexedCachedDataset(IndexedDataset): + def __init__(self, path): + super().__init__(path) + self.cache = None + self.cache_index = {} + + @property + def supports_prefetch(self): + return True + + def prefetch(self, indices): + if all(i in self.cache_index for i in indices): + return + if not self.data_file: + self.read_data(self.path) + indices = sorted(set(indices)) + total_size = 0 + for i in indices: + total_size += self.data_offsets[i + 1] - self.data_offsets[i] + self.cache = np.empty(total_size, dtype=self.dtype) + ptx = 0 + self.cache_index.clear() + for i in indices: + self.cache_index[i] = ptx + size = self.data_offsets[i + 1] - self.data_offsets[i] + a = self.cache[ptx : ptx + size] + self.data_file.seek(self.data_offsets[i] * self.element_size) + self.data_file.readinto(a) + ptx += size + if self.data_file: + # close and delete data file after prefetch so we can pickle + self.data_file.close() + self.data_file = None + + # @lru_cache(maxsize=8) + def __getitem__(self, idx): + if isinstance(idx, int): + i = idx + self.check_index(i) + tensor_size = self.sizes[self.dim_offsets[i] : self.dim_offsets[i + 1]] + a = np.empty(tensor_size, dtype=self.dtype) + ptx = self.cache_index[i] + np.copyto(a, self.cache[ptx : ptx + a.size]) + return a + elif isinstance(idx, slice): + # Hack just to make this work, can optimizer later if necessary + sents = [] + for i in range(*idx.indices(len(self))): + sents.append(self[i]) + return sents + + +class IndexedDatasetBuilder(object): + element_sizes = {np.uint8: 1, np.int8: 1, np.int16: 2, np.int32: 4, np.int64: 8, float: 4, np.double: 8} + + def __init__(self, out_file, dtype=np.int32): + self.out_file = open(out_file, "wb") + self.dtype = dtype + self.data_offsets = [0] + self.dim_offsets = [0] + self.sizes = [] + self.element_size = self.element_sizes[self.dtype] + self.doc_idx = [0] + + def add_item(self, tensor): + bytes = self.out_file.write(np.array(tensor.numpy(), dtype=self.dtype)) + self.data_offsets.append(self.data_offsets[-1] + bytes / self.element_size) + for s in tensor.size(): + self.sizes.append(s) + self.dim_offsets.append(self.dim_offsets[-1] + len(tensor.size())) + + def end_document(self): + self.doc_idx.append(len(self.sizes)) + + def merge_file_(self, another_file): + index = IndexedDataset(another_file) + assert index.dtype == self.dtype + + doc_offset = len(self.sizes) + + begin = self.data_offsets[-1] + for data_offset in index.data_offsets[1:]: + self.data_offsets.append(begin + data_offset) + self.sizes.extend(index.sizes) + + begin = self.dim_offsets[-1] + for dim_offset in index.dim_offsets[1:]: + self.dim_offsets.append(begin + dim_offset) + + self.doc_idx.extend((doc_offset + index.doc_idx)[1:]) + + with open(data_file_path(another_file), "rb") as f: + while True: + data = f.read(1024) + if data: + self.out_file.write(data) + else: + break + + def finalize(self, index_file): + self.out_file.close() + index = open(index_file, "wb") + index.write(b"TNTIDX\x00\x00") + index.write(struct.pack(" list[int]: + return self.tokenizer.tokenize(text) + + def decode(self, tokens: list[int]) -> str: + return self.tokenizer.detokenize(tokens) + + @property + def special_tokens(self) -> dict: + return self.tokenizer._special_tokens + + +class DatasetWriter: + def __init__( + self, + filename_prefix: str, + vocab_size: int, + dataset_impl: str = "mmap", + feature: str = "text", + ): + self.bin_filename = f"{filename_prefix}-{feature}.bin" + self.idx_filename = f"{filename_prefix}-{feature}.idx" + self.builder = indexed_dataset.make_builder(self.bin_filename, impl=dataset_impl, vocab_size=vocab_size) + + def add_item(self, tokenized_item): + self.builder.add_item(torch.IntTensor(tokenized_item)) + + def finalize(self): + self.builder.finalize(self.idx_filename) + + +def format_pairs(pairs: list[str] | tuple[str]) -> tuple[list[str], list[int]]: + assert isinstance(pairs, list) or isinstance(pairs, tuple) + role_names = ("user", "assistant") + role_ids = (1, 2) + return [f"<|im_start|>{role_names[i%2]}\n{pairs[i]}<|im_end|>\n" for i in range(len(pairs))], [ + role_ids[i % 2] for i in range(len(pairs)) + ] + + +def format_sft_entry(entry: DatasetEntrySft) -> tuple[list[str], list[int]]: + turns = [] + roles = [] + if entry.system_message and len(entry.system_message) > 0: + turns.append(f"<|im_start|>system\n{entry.system_message}<|im_end|>\n") + roles.append(IntRole.System.value) # 0 + for m in entry.conversation: + if m.context: + turns.append(f"<|im_start|>context\n{m.context}<|im_end|>\n") + roles.append(IntRole.Context.value) # 3 + if m.role == Role.prompter: + turns.append(f"<|im_start|>user\n{m.text}<|im_end|>\n") + roles.append(IntRole.Prompter.value) # 1 + elif m.role == Role.assistant: + turns.append(f"<|im_start|>assistant\n{m.text}<|im_end|>\n") + roles.append(IntRole.Assistant.value) # 2 + return turns, roles + + +def format_conversation(messages) -> str: + if isinstance(messages, DatasetEntrySft): + return format_sft_entry(messages) + elif isinstance(messages, DatasetEntryLm): + return messages.text, [3] + else: + return format_pairs(messages) + + +def get_dataset_name(d: Dataset): + if isinstance(d, Subset): + inner = d + while isinstance(inner, Subset): + inner = inner.dataset + name = f"Subset of {type(inner).__name__}" + if hasattr(inner, "name"): + name += f" ({inner.name})" + else: + name = type(d).__name__ + if hasattr(d, "name"): + name += f" ({d.name})" + return name + + +class TokenStats: + def __init__(self, name: str, total_samples: int, fraction: float = 1): + self.name = name + self.skipped_samples = 0 + self.skipped_tokens = 0 + self.total_samples = total_samples + self.min_tokens = None + self.max_tokens = 0 + self.accepted_samples = 0 + self.accepted_tokens = 0 + self.fraction = fraction + + @property + def processed_samples(self) -> int: + return self.accepted_samples + self.skipped_samples + + def skip(self, tokens: list[int]) -> None: + self.skipped_samples += 1 + self.skipped_tokens = len(tokens) + + def add(self, tokens: list[int]) -> None: + l = len(tokens) + self.accepted_samples += 1 + self.accepted_tokens += l + if self.min_tokens is None or self.min_tokens > l: + self.min_tokens = l + if self.max_tokens < l: + self.max_tokens = l + + +def tokenize_dataset( + output_dir: Path, + filename_prefix: str, + dataset: Dataset, + encoder: Encoder, + dataset_impl: str, + datasets_config: dict, + max_count: int | None = None, + min_assistant_tokens: int | None = None, + check_tokenization: bool = True, + write_json: bool = False, + seed: int = 42, +): + full_prefix = str(output_dir / filename_prefix) + + token_writer = None + role_writer = None + jsonl_file = None + + per_dataset_stats: list[TokenStats] = [] + cumulative_sizes: list[int] = [] + + rng = np.random.RandomState(seed=seed) + + if isinstance(dataset, ConcatDataset): + datasets = list(dataset.datasets) + + if datasets_config: + dataset_sizes = [len(x) for x in datasets] + fractions = get_dataset_fractions(datasets_config, dataset_sizes, False) + dataset_target_sizes = [int(size * frac) for size, frac in zip(dataset_sizes, fractions)] + else: + dataset_target_sizes = None + + for i in range(len(datasets)): + d = datasets[i] + name = get_dataset_name(d) + frac = 1 + if dataset_target_sizes: + frac = fractions[i] + if dataset_target_sizes[i] < len(d): + # sample subset of dataset + subset_indices = rng.choice(len(d), size=dataset_target_sizes[i], replace=False) + d = Subset(d, subset_indices) + datasets[i] = d + + per_dataset_stats.append(TokenStats(name, len(d), frac)) + + dataset = ConcatDataset(datasets) + cumulative_sizes = dataset.cumulative_sizes + else: + cumulative_sizes = [len(dataset)] + + total_stats = TokenStats("total", len(dataset)) + + try: + token_writer = DatasetWriter( + filename_prefix=full_prefix, + dataset_impl=dataset_impl, + vocab_size=encoder.tokenizer.vocab_size, + feature="text", + ) + + role_writer = DatasetWriter( + filename_prefix=full_prefix, + dataset_impl=dataset_impl, + vocab_size=16, + feature="role", + ) + + jsonl_path = Path(full_prefix + ".jsonl") + if write_json: + jsonl_file = jsonl_path.open("w", encoding="UTF-8") + + subset_index = 0 + for i, messages in enumerate(tqdm(dataset)): + if i >= cumulative_sizes[subset_index]: + subset_index += 1 + + if i > 0 and i % 10000 == 0: + print( + f"Accepted: {total_stats.accepted_samples}/{total_stats.processed_samples} ({total_stats.accepted_samples/total_stats.processed_samples:.1%})" + ) + + turns, turn_roles = format_conversation(messages) + + tokens = [] + role_lables = [] + num_assistant_tokens = 0 + for t, r in zip(turns, turn_roles): + turn_tokens = encoder.encode_text(t) + turn_role = [r] * len(turn_tokens) + tokens.extend(turn_tokens) + if r == IntRole.Assistant: + num_assistant_tokens += len(turn_tokens) + role_lables.extend(turn_role) + + if min_assistant_tokens is not None and num_assistant_tokens < min_assistant_tokens: + total_stats.skip(tokens) + per_dataset_stats[subset_index].skip(tokens) + continue + + if check_tokenization: + x = encoder.encode_text("".join(turns)) + assert x == tokens and len(tokens) == len(role_lables) + + token_writer.add_item(tokens) + role_writer.add_item(role_lables) + + # update stats + total_stats.add(tokens) + per_dataset_stats[subset_index].add(tokens) + + if jsonl_file: + json.dump({"text": "".join(turns)}, jsonl_file) + jsonl_file.write("\n") + + if max_count and total_stats.accepted_samples >= max_count: + break + finally: + if token_writer: + token_writer.finalize() + if role_writer: + role_writer.finalize() + if jsonl_file: + jsonl_file.close() + + per_dataset_stats.append(total_stats) + + stats_path = Path(full_prefix + "_stats.txt") + with stats_path.open("w", encoding="UTF-8") as stats_file: + for f in (None, stats_file): + print(f"\n# Stats for {full_prefix}*\n", file=f) + + for stats in per_dataset_stats: + print(f"## Stats for '{stats.name}' ({stats.total_samples} samples ({stats.fraction:.1%}))", file=f) + print("-----------------", file=f) + print( + f" Accepted: {stats.accepted_samples}/{stats.processed_samples} ({stats.accepted_samples/stats.processed_samples:.1%})", + file=f, + ) + print(f" Accepted tokens: {stats.accepted_tokens}", file=f) + print( + f" Skipped: {stats.skipped_samples} ({stats.skipped_samples/stats.processed_samples:.1%})", file=f + ) + print(f" Min tokens per sample: {stats.min_tokens}", file=f) + print(f" Max tokens per sample: {stats.max_tokens}", file=f) + print(f" Avg tokens per sample: {stats.accepted_tokens/stats.accepted_samples}", file=f) + print("-----------------\n", file=f) + + +def parse_args(): + parser = argparse.ArgumentParser( + prog="pretokenize.py", description="Tokenize datamixes for LLama2/Falcon fine-tuning with Megatron-LLM." + ) + group = parser.add_argument_group(title="configuration") + group.add_argument( + "--configs", + nargs="+", + required=True, + help="Configurations sections to apply (read from YAML, multiple can be specified).", + ) + group.add_argument( + "--output_dir", + type=str, + help="Path to output directory", + ) + group.add_argument( + "--write_json", + action="store_true", + help="Generate a JSONL file with the formatted dialogues (key='text').", + ) + group.add_argument( + "--compress", + action="store_true", + help="Generate a .tar.gz file of the output directory.", + ) + + args, remaining = parser.parse_known_args() + + # load yaml configurations + conf = {} + configs = read_yamls("./configs") + conf.update(configs["defaults"]) + try: + for name in args.configs: + if "," in name: + for n in name.split(","): + conf.update(configs[n]) + else: + conf.update(configs[name]) + except KeyError as e: + print(f'Error: Section "{e.args[0]}" not found in YAML configuration files.') + exit(1) + + # override yaml args + for k, v in vars(args).items(): + if k == "configs" or v is None: + continue + conf[k] = v + + parser = argparse.ArgumentParser() + for key, value in conf.items(): + type_ = type(value) if value is not None else str + if type_ == bool: + type_ = _strtobool + parser.add_argument(f"--{key}", type=type_, default=value) + # Allow --no-{key} to remove a configuration value + parser.add_argument(f"--no-{key}", dest=key, action="store_const", const=None) + parser.add_argument( + "--max_count", + type=int, + help="Limit number of train/eval examples to process (debug)", + ) + + args = parser.parse_args(remaining) + args.keep_empty = False + args.rank = 0 + args.vocab_extra_ids = 0 + args.make_vocab_size_divisible_by = 128 + args.tensor_model_parallel_size = 1 + args.new_tokens = True + + return args + + +def main(): + """ + Example usage: `python __main__.py --output_dir output--configs oasst_top1 llama2` + """ + args = parse_args() + print("Configuration:") + for k, v in vars(args).items(): + print(f"{k}: {v}") + + # initialize random states for reproducibility + random.seed(args.rng_seed) + np.random.seed(args.rng_seed) + torch.manual_seed(args.rng_seed) + + print("Building encoder") + encoder = Encoder(args) + + tokenizer_check_input = "<|im_start|>system\nsystem message<|im_end|>\n<|im_start|>user\nprompt<|im_end|><|im_start|>assistant\nreply<|im_end|>\n" + tokenizer_check_output = encoder.encode_text(tokenizer_check_input) + print("Tokenizer check:") + print("Input:", tokenizer_check_input.replace("\n", r"\n")) + print("Output:", tokenizer_check_output) + print(f"Vocab size: {encoder.tokenizer.vocab_size}") + + output_dir = Path(args.output_dir + args.output_dir_suffix) + print(f"Output dir: {output_dir} (exists: {output_dir.exists()})") + + train, evals = get_dataset(args) + + # show dataset stats + print("Training dataset sizes (before sampling):") + total = len(train) + for d in train.datasets: + name = get_dataset_name(d) + print(f"{name}: {len(d)} ({len(d) / total:.2%})") + + output_dir.mkdir(parents=True, exist_ok=True) + + fn = output_dir / "special_tokens.json" + with fn.open("w", encoding="UTF-8") as f: + json.dump(encoder.special_tokens, f) + + val = ConcatDataset(evals.values()) + for split_name, ds in zip(["train", "val"], [train, val]): + datasets_config = args.datasets if split_name == "train" else None + tokenize_dataset( + output_dir=output_dir, + filename_prefix=f"{args.filename_prefix}-{split_name}", + dataset=ds, + encoder=encoder, + dataset_impl=args.dataset_impl, + datasets_config=datasets_config, + max_count=args.max_count, + min_assistant_tokens=args.min_assistant_tokens, + write_json=args.write_json, + seed=args.rng_seed, + ) + + if args.compress: + run(f"tar -czvf {output_dir}.tar.gz {output_dir}", shell=True, check=True) + + +if __name__ == "__main__": + main() diff --git a/model/pretokenizer/requirements.txt b/model/pretokenizer/requirements.txt new file mode 100644 index 000000000..2ccf6c1ed --- /dev/null +++ b/model/pretokenizer/requirements.txt @@ -0,0 +1,5 @@ +numpy>=1.22.4 +sentencepiece==0.1.99 +torch>=2.0.0 +tqdm==4.65.0 +transformers==4.31.0 diff --git a/model/pretokenizer/tokenizer.py b/model/pretokenizer/tokenizer.py new file mode 100644 index 000000000..5ee75a32a --- /dev/null +++ b/model/pretokenizer/tokenizer.py @@ -0,0 +1,331 @@ +# copied from https://github.com/epfLLM/Megatron-LLM/blob/main/megatron/tokenizer/tokenizer.py +# (only keeping _FalconTokenizer & _SentencePieceTokenizer) + +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + +"""Megatron tokenizers.""" + +from abc import ABC, abstractmethod + + +def build_tokenizer(args): + """Initialize tokenizer.""" + if args.rank == 0: + print("> building {} tokenizer ...".format(args.tokenizer_type), flush=True) + + if args.tokenizer_type not in {"SentencePieceTokenizer", "FalconTokenizer"}: + assert args.vocab_file is not None + + # Select and instantiate the tokenizer. + if args.tokenizer_type == "SentencePieceTokenizer": + assert args.vocab_file is not None + tokenizer = _SentencePieceTokenizer( + args.vocab_file, + vocab_extra_ids=args.vocab_extra_ids, + vocab_extra_ids_list=args.vocab_extra_ids_list, + new_tokens=args.new_tokens, + ) + elif args.tokenizer_type == "FalconTokenizer": + tokenizer = _FalconTokenizer(vocab_extra_ids_list=args.vocab_extra_ids_list, new_tokens=args.new_tokens) + else: + raise NotImplementedError("{} tokenizer is not " "implemented.".format(args.tokenizer_type)) + + # Add vocab size. + args.padded_vocab_size = _vocab_size_with_padding(tokenizer.vocab_size, args) + + return tokenizer + + +def _vocab_size_with_padding(orig_vocab_size, args): + """Pad vocab size so it is divisible by model parallel size and + still having GPU friendly size.""" + + after = orig_vocab_size + multiple = args.make_vocab_size_divisible_by * args.tensor_model_parallel_size + while (after % multiple) != 0: + after += 1 + if args.rank == 0: + print( + " > padded vocab (size: {}) with {} dummy tokens " + "(new size: {})".format(orig_vocab_size, after - orig_vocab_size, after), + flush=True, + ) + return after + + +class AbstractTokenizer(ABC): + """Abstract class for tokenizer.""" + + def __init__(self, name): + self.name = name + super().__init__() + + @property + @abstractmethod + def vocab_size(self): + pass + + @property + @abstractmethod + def vocab(self): + """Dictionary from vocab text token to id token.""" + pass + + @property + @abstractmethod + def inv_vocab(self): + """Dictionary from vocab id token to text token.""" + pass + + @abstractmethod + def tokenize(self, text): + pass + + def detokenize(self, token_ids): + raise NotImplementedError("detokenizer is not implemented for {} " "tokenizer".format(self.name)) + + @property + def cls(self): + raise NotImplementedError("CLS is not provided for {} " "tokenizer".format(self.name)) + + @property + def sep(self): + raise NotImplementedError("SEP is not provided for {} " "tokenizer".format(self.name)) + + @property + def pad(self): + raise NotImplementedError("PAD is not provided for {} " "tokenizer".format(self.name)) + + @property + def eod(self): + raise NotImplementedError("EOD is not provided for {} " "tokenizer".format(self.name)) + + @property + def mask(self): + raise NotImplementedError("MASK is not provided for {} " "tokenizer".format(self.name)) + + +class _FalconTokenizer(AbstractTokenizer): + """Wrapper of huggingface tokenizer.""" + + def __init__(self, vocab_extra_ids_list=None, new_tokens=True): + name = "FalconTokenizer" + super().__init__(name) + from transformers import AutoTokenizer + + self.tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-40b") + + if vocab_extra_ids_list and new_tokens: + special_tokens = self.tokenizer.additional_special_tokens + vocab_extra_ids_list.split(",") + self.tokenizer.add_special_tokens({"additional_special_tokens": special_tokens}) + self._special_tokens = {tok: self.vocab[tok] for tok in special_tokens} + else: + self._special_tokens = {} + + self._inv_vocab = {idx: token for token, idx in self.tokenizer.vocab.items()} + + @property + def vocab_size(self): + return len(self.tokenizer.vocab) + + @property + def vocab(self): + return self.tokenizer.vocab + + def tokenize(self, text): + return self.tokenizer.encode(text) + + def detokenize(self, token_ids): + return self.tokenizer.decode(token_ids) + + @property + def inv_vocab(self): + return self._inv_vocab + + @property + def eod(self): + return self.eos_token_id + + @property + def pad(self): + return self.eos_token_id + + @property + def eos_token_id(self): + return self.tokenizer.eos_token_id + + +class _SentencePieceTokenizer(AbstractTokenizer): + """SentencePieceTokenizer-Megatron wrapper""" + + def __init__(self, model_file, vocab_extra_ids=0, vocab_extra_ids_list=None, new_tokens=True): + name = "SentencePieceTokenizer" + super().__init__(name) + + import sentencepiece + + self._tokenizer = sentencepiece.SentencePieceProcessor(model_file=model_file) + + self._initalize(vocab_extra_ids, vocab_extra_ids_list, new_tokens) + + def _initalize(self, vocab_extra_ids, vocab_extra_ids_list, new_tokens): + self._vocab = {} + self._inv_vocab = {} + + self._special_tokens = {} + self._inv_special_tokens = {} + + self._t5_tokens = [] + + for i in range(len(self._tokenizer)): + t = self._tokenizer.id_to_piece(i) + self._inv_vocab[i] = t + self._vocab[t] = i + + def _add_special_token(t): + if t not in self.vocab and not new_tokens: + return + if t not in self._vocab: + next_id = len(self._vocab) + self._vocab[t] = next_id + self._inv_vocab[next_id] = t + self._special_tokens[t] = self._vocab[t] + self._inv_special_tokens[self._vocab[t]] = t + + _add_special_token("") + self._cls_id = self._vocab.get("") + _add_special_token("") + self._sep_id = self._vocab.get("") + _add_special_token("") + self._eod_id = self._vocab.get("") + _add_special_token("") + self._mask_id = self._vocab.get("") + + pad_id = self._tokenizer.pad_id() + try: + pad_token = self._tokenizer.id_to_piece(pad_id) + except IndexError: + pad_token = "" + _add_special_token(pad_token) + self._pad_id = self._vocab.get(pad_token) + + bos_id = self._tokenizer.bos_id() + try: + bos_token = self._tokenizer.id_to_piece(bos_id) + except IndexError: + bos_token = "" + _add_special_token(bos_token) + self._bos_id = self._vocab.get(bos_token) + + eos_id = self._tokenizer.eos_id() + try: + eos_token = self._tokenizer.id_to_piece(eos_id) + except IndexError: + eos_token = "" + _add_special_token(eos_token) + self._eos_id = self._vocab.get(eos_token) + + for i in range(vocab_extra_ids): + t = "".format(i) + _add_special_token(t) + self._t5_tokens += [t] + if vocab_extra_ids_list: + for t in vocab_extra_ids_list.split(","): + _add_special_token(t) + print("Special tokens: {}".format(self._special_tokens)) + + @property + def vocab_size(self): + return len(self._vocab) + + @property + def vocab(self): + return self._vocab + + @property + def inv_vocab(self): + return self._inv_vocab + + # From: + # https://github.com/NVIDIA/NeMo/blob/c8fa217e811d60d11d014827c7f3845ff6c99ae7/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py#L89 + def tokenize(self, text): + ids = [] + idx = 0 + + while 1: + indices = {} + for token in self._special_tokens: + try: + indices[token] = text[idx:].index(token) + except ValueError: + continue + if len(indices) == 0: + break + + next_token = min(indices, key=indices.get) + next_idx = idx + indices[next_token] + + ids.extend(self._tokenizer.encode_as_ids(text[idx:next_idx])) + ids.append(self._special_tokens[next_token]) + idx = next_idx + len(next_token) + + ids.extend(self._tokenizer.encode_as_ids(text[idx:])) + return ids + + # From: + # https://github.com/NVIDIA/NeMo/blob/c8fa217e811d60d11d014827c7f3845ff6c99ae7/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py#L125 + def detokenize(self, ids): + text = "" + last_i = 0 + + for i, id in enumerate(ids): + if id in self._inv_special_tokens: + text += self._tokenizer.decode_ids(ids[last_i:i]) + " " + text += self._inv_special_tokens[id] + " " + last_i = i + 1 + text += self._tokenizer.decode_ids(ids[last_i:]) + return text.strip() + + @property + def cls(self): + return self._cls_id + + @property + def sep(self): + return self._sep_id + + @property + def pad(self): + return self._pad_id + + @property + def bos_token_id(self): + return self._bos_id + + @property + def bos(self): + return self._bos_id + + @property + def eod(self): + if self._eod_id is not None: + return self._eod_id + return self._eos_id # in case noe eod we can patch this up with an eos + + @property + def eos_token_id(self): + if self._eod_id is not None: + return self._eod_id + return self._eos_id + + @property + def eos(self): + return self._eos_id + + @property + def mask(self): + return self._mask_id + + @property + def additional_special_tokens_ids(self): + return [self.vocab[k] for k in self._t5_tokens] diff --git a/model/pyproject.toml b/model/pyproject.toml index a0e9931ba..d03a1d12a 100644 --- a/model/pyproject.toml +++ b/model/pyproject.toml @@ -42,7 +42,7 @@ tests = [ ] [tool.setuptools] -py-modules = ["model_training"] +packages = ["model_training"] [tool.black] line-length = 120 diff --git a/oasst-data/pyproject.toml b/oasst-data/pyproject.toml index 418517648..2f9a1a9d1 100644 --- a/oasst-data/pyproject.toml +++ b/oasst-data/pyproject.toml @@ -6,7 +6,7 @@ authors = [ { name = "LAION-AI", email = "contact@laion.ai" } ] dependencies = [ - "pydantic>=1.10.4", + "pydantic==1.10.7", "loguru==0.6.0", "datasets>=2.12.0" ] diff --git a/oasst-shared/oasst_shared/model_configs.py b/oasst-shared/oasst_shared/model_configs.py index 13b78e17b..bd85e4461 100644 --- a/oasst-shared/oasst_shared/model_configs.py +++ b/oasst-shared/oasst_shared/model_configs.py @@ -145,4 +145,14 @@ def compat_hash(self) -> str: max_total_length=2048, quantized=True, ), + "OA_SFT_Llama2_70B_10": ModelConfig( + model_id="OpenAssistant/llama2-70b-oasst-sft-v10", + max_input_length=3072, + max_total_length=4096, + ), + "OA_SFT_CodeLlama_13B_10": ModelConfig( + model_id="OpenAssistant/codellama-13b-oasst-sft-v10", + max_input_length=8192, + max_total_length=12288, + ), } diff --git a/oasst-shared/oasst_shared/schemas/inference.py b/oasst-shared/oasst_shared/schemas/inference.py index d3afdeaba..c4110285d 100644 --- a/oasst-shared/oasst_shared/schemas/inference.py +++ b/oasst-shared/oasst_shared/schemas/inference.py @@ -208,6 +208,8 @@ class WorkParameters(pydantic.BaseModel): default_factory=make_seed, ) system_prompt: str | None = None + user_profile: str | None = None + user_response_instructions: str | None = None plugins: list[PluginEntry] = pydantic.Field(default_factory=list[PluginEntry]) plugin_max_depth: int = 4 diff --git a/oasst-shared/pyproject.toml b/oasst-shared/pyproject.toml index 85f3cee84..baee4f9e1 100644 --- a/oasst-shared/pyproject.toml +++ b/oasst-shared/pyproject.toml @@ -6,7 +6,7 @@ authors = [ { name = "LAION-AI", email = "contact@laion.ai" } ] dependencies = [ - "pydantic==1.10.4", + "pydantic==1.10.7", "aiohttp==3.8.3", "aiohttp[speedups]", "loguru==0.6.0", diff --git a/website/package-lock.json b/website/package-lock.json index 77ebdc66b..b731e7e50 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -34,7 +34,7 @@ "chart.js": "^4.3.0", "clsx": "^1.2.1", "date-fns": "^2.30.0", - "eslint": "^8.41.0", + "eslint": "^8.46.0", "eslint-config-next": "^13.4.6", "eslint-plugin-simple-import-sort": "^10.0.0", "focus-visible": "^5.2.0", @@ -53,7 +53,7 @@ "react-cookie": "^4.1.1", "react-dom": "^18.2.0", "react-feature-flags": "^1.0.0", - "react-hook-form": "^7.44.3", + "react-hook-form": "^7.45.4", "react-i18next": "^12.2.2", "react-markdown": "^8.0.7", "react-syntax-highlighter": "^15.5.0", @@ -79,7 +79,7 @@ "@storybook/addon-links": "^7.0.20", "@storybook/nextjs": "^7.0.22", "@storybook/react": "^7.0.22", - "@storybook/testing-library": "^0.1.0", + "@storybook/testing-library": "^0.2.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", "@types/accept-language-parser": "^1.5.3", @@ -99,7 +99,7 @@ "msw": "^1.2.1", "msw-storybook-addon": "^1.8.0", "path-browserify": "^1.0.1", - "pino-pretty": "^10.0.0", + "pino-pretty": "^10.2.0", "prettier": "^2.8.8", "prisma": "^4.14.0", "storybook": "^7.0.9", @@ -109,6 +109,14 @@ "web-streams-polyfill": "^3.2.1" } }, + "node_modules/@aashutoshrathi/word-wrap": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", + "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/@adobe/css-tools": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.2.0.tgz", @@ -4210,21 +4218,21 @@ } }, "node_modules/@eslint-community/regexpp": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.5.1.tgz", - "integrity": "sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ==", + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.6.2.tgz", + "integrity": "sha512-pPTNuaAG3QMH+buKyBIGJs3g/S5y0caxw0ygM3YyE6yJFySwiGGSzA+mM3KJ8QQvzeLh3blwgSonkFjgQdxzMw==", "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } }, "node_modules/@eslint/eslintrc": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz", - "integrity": "sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.1.tgz", + "integrity": "sha512-9t7ZA7NGGK8ckelF0PQCfcxIUzs1Md5rrO6U/c+FIQNanea5UZC0wqKXH4vHBccmu4ZJgZ2idtPeW7+Q2npOEA==", "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", - "espree": "^9.5.2", + "espree": "^9.6.0", "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", @@ -4281,9 +4289,9 @@ } }, "node_modules/@eslint/js": { - "version": "8.41.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.41.0.tgz", - "integrity": "sha512-LxcyMGxwmTh2lY9FwHPGWOHmYFCZvbrFCBZL4FzSSsxsRPuhrYUg/49/0KDfW8tnIEaEHtfmn6+NPN+1DqaNmA==", + "version": "8.46.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.46.0.tgz", + "integrity": "sha512-a8TLtmPi8xzPkCbp/OGFUo5yhRkHM2Ko9kOWP4znJr0WAhWyThaw3PnwX4vOTWOAMsV2uRt32PPDcEz63esSaA==", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } @@ -4311,9 +4319,9 @@ "dev": true }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.8", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", - "integrity": "sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g==", + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", + "integrity": "sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ==", "dependencies": { "@humanwhocodes/object-schema": "^1.2.1", "debug": "^4.1.1", @@ -12152,18 +12160,87 @@ } }, "node_modules/@storybook/testing-library": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@storybook/testing-library/-/testing-library-0.1.0.tgz", - "integrity": "sha512-g947f4LJZw3IluBhysMKLJXByAFiSxnGuooENqU+ZPt/GTrz1I9GDBlhmoTJahuFkVbwHvziAl/8riY2Re921g==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@storybook/testing-library/-/testing-library-0.2.0.tgz", + "integrity": "sha512-Ff6jNnrsosmDshgCf0Eb5Cz7IA34p/1Ps5N3Kp3598kfXpBSccSkQQvVFUXC3kIHw/isIXWPqntZuKqnWUz7Gw==", "dev": true, "dependencies": { - "@storybook/client-logger": "^7.0.0-beta.0 || ^7.0.0-rc.0 || ^7.0.0", - "@storybook/instrumenter": "^7.0.0-beta.0 || ^7.0.0-rc.0 || ^7.0.0", - "@testing-library/dom": "^8.3.0", - "@testing-library/user-event": "^13.2.1", + "@testing-library/dom": "^9.0.0", + "@testing-library/user-event": "^14.0.0", "ts-dedent": "^2.2.0" } }, + "node_modules/@storybook/testing-library/node_modules/@testing-library/dom": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-9.3.1.tgz", + "integrity": "sha512-0DGPd9AR3+iDTjGoMpxIkAsUihHZ3Ai6CneU6bRRrffXMgzCdlNk43jTrD2/5LT6CBb3MWTP8v510JzYtahD2w==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/runtime": "^7.12.5", + "@types/aria-query": "^5.0.1", + "aria-query": "5.1.3", + "chalk": "^4.1.0", + "dom-accessibility-api": "^0.5.9", + "lz-string": "^1.5.0", + "pretty-format": "^27.0.2" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@storybook/testing-library/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@storybook/testing-library/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/@storybook/testing-library/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@storybook/testing-library/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@storybook/theming": { "version": "7.0.9", "resolved": "https://registry.npmjs.org/@storybook/theming/-/theming-7.0.9.tgz", @@ -12411,15 +12488,12 @@ } }, "node_modules/@testing-library/user-event": { - "version": "13.5.0", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-13.5.0.tgz", - "integrity": "sha512-5Kwtbo3Y/NowpkbRuSepbyMFkZmHgD+vPzYB/RJ4oxt5Gj/avFFBYjhw27cqSVPVw/3a67NK1PbiIr9k4Gwmdg==", + "version": "14.4.3", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.4.3.tgz", + "integrity": "sha512-kCUc5MEwaEMakkO5x7aoD+DLi02ehmEM2QCGWvNqAS1dV/fAvORWEjnjsEIvml59M7Y5kCkWN6fCCyPOe8OL6Q==", "dev": true, - "dependencies": { - "@babel/runtime": "^7.12.5" - }, "engines": { - "node": ">=10", + "node": ">=12", "npm": ">=6" }, "peerDependencies": { @@ -17832,26 +17906,26 @@ } }, "node_modules/eslint": { - "version": "8.41.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.41.0.tgz", - "integrity": "sha512-WQDQpzGBOP5IrXPo4Hc0814r4/v2rrIsB0rhT7jtunIalgg6gYXWhRMOejVO8yH21T/FGaxjmFjBMNqcIlmH1Q==", + "version": "8.46.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.46.0.tgz", + "integrity": "sha512-cIO74PvbW0qU8e0mIvk5IV3ToWdCq5FYG6gWPHHkx6gNdjlbAYvtfHmlCMXxjcoVaIdwy/IAt3+mDkZkfvb2Dg==", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", - "@eslint-community/regexpp": "^4.4.0", - "@eslint/eslintrc": "^2.0.3", - "@eslint/js": "8.41.0", - "@humanwhocodes/config-array": "^0.11.8", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.1", + "@eslint/js": "^8.46.0", + "@humanwhocodes/config-array": "^0.11.10", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", - "ajv": "^6.10.0", + "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", "debug": "^4.3.2", "doctrine": "^3.0.0", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.2.0", - "eslint-visitor-keys": "^3.4.1", - "espree": "^9.5.2", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.2", + "espree": "^9.6.1", "esquery": "^1.4.2", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -17861,7 +17935,6 @@ "globals": "^13.19.0", "graphemer": "^1.4.0", "ignore": "^5.2.0", - "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "is-path-inside": "^3.0.3", @@ -17871,9 +17944,8 @@ "lodash.merge": "^4.6.2", "minimatch": "^3.1.2", "natural-compare": "^1.4.0", - "optionator": "^0.9.1", + "optionator": "^0.9.3", "strip-ansi": "^6.0.1", - "strip-json-comments": "^3.1.0", "text-table": "^0.2.0" }, "bin": { @@ -18267,9 +18339,9 @@ } }, "node_modules/eslint-visitor-keys": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz", - "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.2.tgz", + "integrity": "sha512-8drBzUEyZ2llkpCA67iYrgEssKDUu68V8ChqqOfFupIaG/LCVPUT+CoGJpT77zJprs4T/W7p07LP7zAIMuweVw==", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, @@ -18312,9 +18384,9 @@ } }, "node_modules/eslint/node_modules/eslint-scope": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.0.tgz", - "integrity": "sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==", + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" @@ -18360,16 +18432,16 @@ } }, "node_modules/eslint/node_modules/optionator": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", - "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", + "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", "dependencies": { + "@aashutoshrathi/word-wrap": "^1.2.3", "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.3" + "type-check": "^0.4.0" }, "engines": { "node": ">= 0.8.0" @@ -18398,11 +18470,11 @@ } }, "node_modules/espree": { - "version": "9.5.2", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz", - "integrity": "sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==", + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", "dependencies": { - "acorn": "^8.8.0", + "acorn": "^8.9.0", "acorn-jsx": "^5.3.2", "eslint-visitor-keys": "^3.4.1" }, @@ -18414,9 +18486,9 @@ } }, "node_modules/espree/node_modules/acorn": { - "version": "8.8.2", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", - "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", + "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", "bin": { "acorn": "bin/acorn" }, @@ -26914,9 +26986,9 @@ } }, "node_modules/pino-pretty": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.0.0.tgz", - "integrity": "sha512-zKFjYXBzLaLTEAN1ayKpHXtL5UeRQC7R3lvhKe7fWs7hIVEjKGG/qIXwQt9HmeUp71ogUd/YcW+LmMwRp4KT6Q==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.2.0.tgz", + "integrity": "sha512-tRvpyEmGtc2D+Lr3FulIZ+R1baggQ4S3xD2Ar93KixFEDx6SEAUP3W5aYuEw1C73d6ROrNcB2IXLteW8itlwhA==", "dev": true, "dependencies": { "colorette": "^2.0.7", @@ -28150,9 +28222,9 @@ } }, "node_modules/react-hook-form": { - "version": "7.44.3", - "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.44.3.tgz", - "integrity": "sha512-/tHId6p2ViAka1wECMw8FEPn/oz/w226zehHrJyQ1oIzCBNMIJCaj6ZkQcv+MjDxYh9MWR7RQic7Qqwe4a5nkw==", + "version": "7.45.4", + "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.45.4.tgz", + "integrity": "sha512-HGDV1JOOBPZj10LB3+OZgfDBTn+IeEsNOKiq/cxbQAIbKaiJUe/KV8DBUzsx0Gx/7IG/orWqRRm736JwOfUSWQ==", "engines": { "node": ">=12.22.0" }, @@ -32473,6 +32545,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true, "engines": { "node": ">=0.10.0" } diff --git a/website/package.json b/website/package.json index d5487970b..f3398be8c 100644 --- a/website/package.json +++ b/website/package.json @@ -55,7 +55,7 @@ "chart.js": "^4.3.0", "clsx": "^1.2.1", "date-fns": "^2.30.0", - "eslint": "^8.41.0", + "eslint": "^8.46.0", "eslint-config-next": "^13.4.6", "eslint-plugin-simple-import-sort": "^10.0.0", "focus-visible": "^5.2.0", @@ -74,7 +74,7 @@ "react-cookie": "^4.1.1", "react-dom": "^18.2.0", "react-feature-flags": "^1.0.0", - "react-hook-form": "^7.44.3", + "react-hook-form": "^7.45.4", "react-i18next": "^12.2.2", "react-markdown": "^8.0.7", "react-syntax-highlighter": "^15.5.0", @@ -100,7 +100,7 @@ "@storybook/addon-links": "^7.0.20", "@storybook/nextjs": "^7.0.22", "@storybook/react": "^7.0.22", - "@storybook/testing-library": "^0.1.0", + "@storybook/testing-library": "^0.2.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", "@types/accept-language-parser": "^1.5.3", @@ -120,7 +120,7 @@ "msw": "^1.2.1", "msw-storybook-addon": "^1.8.0", "path-browserify": "^1.0.1", - "pino-pretty": "^10.0.0", + "pino-pretty": "^10.2.0", "prettier": "^2.8.8", "prisma": "^4.14.0", "storybook": "^7.0.9", diff --git a/website/prisma/migrations/20230805220637_paperack/migration.sql b/website/prisma/migrations/20230805220637_paperack/migration.sql new file mode 100644 index 000000000..848c306b2 --- /dev/null +++ b/website/prisma/migrations/20230805220637_paperack/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "User" ADD COLUMN "paperackName" TEXT NOT NULL DEFAULT '', +ADD COLUMN "paperackYes" BOOLEAN NOT NULL DEFAULT false; diff --git a/website/prisma/schema.prisma b/website/prisma/schema.prisma index f59b8e9fd..0aa732bad 100644 --- a/website/prisma/schema.prisma +++ b/website/prisma/schema.prisma @@ -43,6 +43,8 @@ model User { image String? isNew Boolean @default(true) role String @default("general") + paperackYes Boolean @default(false) + paperackName String @default("") accounts Account[] sessions Session[] diff --git a/website/public/images/logos/logo_192x192.png b/website/public/images/logos/logo_192x192.png new file mode 100644 index 000000000..8b09a8272 Binary files /dev/null and b/website/public/images/logos/logo_192x192.png differ diff --git a/website/public/locales/de/chat.json b/website/public/locales/de/chat.json index b99661a80..ddf3341b6 100644 --- a/website/public/locales/de/chat.json +++ b/website/public/locales/de/chat.json @@ -6,6 +6,7 @@ "delete_chat": "Chat löschen", "delete_confirmation": "Sind Sie sicher, dass Sie den Chat löschen wollen?", "delete_confirmation_detail": "Wenn Sie diesen Chat löschen, ist er nicht mehr Teil unserer Daten und wir können ihn nicht mehr zur Verbesserung unserer Modelle verwenden. Bitte nehmen Sie sich die Zeit, Antworten in anderen Chats hoch- und runterzustufen, um uns zu helfen, Open Assistant zu verbessern!", + "drafts_generating_notify": "Die Drafts werden noch erstellt. Bitte warten Sie...", "edit_plugin": "Plugin bearbeiten", "empty": "(leer)", "input_placeholder": "Fragen Sie, was Sie wollen...", @@ -18,7 +19,8 @@ "button": "Keine Trainingsdaten zulassen", "success_message": "Sie haben sich gegen die Verwendung von Trainingsdaten entschieden.", "dialog": { - "title": "Trainingsdaten nicht mehr senden." + "title": "Trainingsdaten nicht mehr senden.", + "description": "Wenn Sie bestätigen, wird dieser Chat nicht zur Verbesserung unseres Modells verwendet. Bitte nehmen Sie sich die Zeit, Antworten in anderen Chats zu bewerten, um uns zu helfen, Open Assistant zu verbessern!" } }, "parameter_description": { @@ -55,5 +57,10 @@ "preset_name_placeholder": "Namen eingeben", "feedback_message": "Wie war ich? Dein Feedback macht mich besser!", "feedback_action_great": "Gut", - "feedback_action_poor": "Könnte besser sein" + "feedback_action_poor": "Könnte besser sein", + "custom_instructions": "Benutzerdefinierte Anweisungen", + "custom_instructions_user_profile": "Welche Informationen sollte Open-Assistant über Sie haben, um die Antworten noch besser zu machen?", + "custom_instructions_response_instructions": "Wie soll der Open-Assistant mit Ihnen chatten?", + "custom_instructions_user_profile_placeholder": "Führen Sie einige Ihrer Wünsche auf.\nBeschreiben Sie Ihre Hobbys und Interessen.\nBeschreiben Sie Ihren Wohnort.\nWelchen Beruf üben Sie aus?\nWelche Themen könnten Sie ausführlich diskutieren?", + "custom_instructions_response_instructions_placeholder": "Soll der Open-Assistant seine Meinung äußern oder neutral bleiben?\nBestimmen Sie den gewünschten Formalitätsgrad für die Antworten des Open-Assistant.\nWie soll der Open-Assistant Sie ansprechen?\nBestimmen Sie die gewünschte Länge der Antworten." } diff --git a/website/public/locales/de/common.json b/website/public/locales/de/common.json index a9ccb9b3a..94aaa6f18 100644 --- a/website/public/locales/de/common.json +++ b/website/public/locales/de/common.json @@ -2,6 +2,7 @@ "about": "Über", "account_settings": "Konto", "admin_dashboard": "Admin-Dashboard", + "add": "Hinzufügen", "back_to_dashboard": "Zum Dashboard zurückkehren", "bug_description": "Wenn Sie Daten beitragen wollten aber hier gelandet sind, melden Sie bitte einen Fehler.", "bug_button": "Fehler melden", @@ -35,12 +36,16 @@ "no": "Nein", "output": "Ausgabe", "parameters": "Parameter", + "plain_text": "Klartext", "privacy_policy": "Datenschutz-Bestimmungen", "prompt": "Prompt", "report_a_bug": "Einen Fehler melden", "retry": "Erneut versuchen", "review": "Review", + "save": "Sichern", "send": "Senden", + "show_less": "weniger anzeigen", + "show_more": "Mehr anzeigen", "sorry_404": "Sorry, die Seite nach der Sie suchen existiert nicht.", "sign_in": "Anmelden", "sign_out": "Abmelden", diff --git a/website/public/locales/de/index.json b/website/public/locales/de/index.json index ed0197e45..7ec9f37e4 100644 --- a/website/public/locales/de/index.json +++ b/website/public/locales/de/index.json @@ -1,6 +1,7 @@ { "blurb": "Wir glauben, dass wir etwas Revolutionäres erschaffen können.", "blurb1": "Genauso wie Stable Diffusion es ermöglicht hat, Kunst und Bilder auf neue Weise zu erschaffen, wollen wir die Welt verbessern, indem wir eine erstklassige Konversations-KI bereitstellen.", + "chat_with_our_assistant": "Chatte mit unserem Assistenten", "description": "Konversations-KI für alle. Ein Open-Source-Projekt zur Erstellung eines Chat-fähigen GPT LLM, das von LAION und Mitwirkenden auf der ganzen Welt betrieben wird.", "faq_items": { "q0": "Was ist der aktuelle Stand des Projekts?", diff --git a/website/public/locales/de/leaderboard.json b/website/public/locales/de/leaderboard.json index 2692e3a7a..75cb1141b 100644 --- a/website/public/locales/de/leaderboard.json +++ b/website/public/locales/de/leaderboard.json @@ -1,18 +1,39 @@ { + "accepted": "↪ Akzeptiert", + "accepted_prompts": "Akzeptierte Prompts", "daily": "Täglich", + "day": "Day", + "good_rankings": "Gute Rankings", "label": "Labels", + "labels_full": "Labels (voll)", + "labels_simple": "Labels (einfach)", "last_updated_at": "Zuletzt aktualisiert: {{val, datetime}}", "leaderboard": "Bestenliste", + "level_progress_message": "Mit einer Gesamtpunktzahl von {{score}} hast du die Stufe {{level,number,integer}} erreicht!", + "month": "Monat", "monthly": "Monatlich", "next": "Nächste", + "no_email": "(keine E-mail)", + "no_username": "(Kein Benutzername)", "overall": "Gesamt", "previous": "Vorherige", "prompt": "Prompts", "rank": "Rang", + "rankings": "Rankings", + "reached_max_level": "Sie haben die maximale Stufe erreicht, vielen Dank für Ihre harte Arbeit!", + "replies_assistant": "Antworten als Assistent", + "replies_prompter": "Antworten als Prompter", "reply": "Antworten", + "reply_ranked_1": "Antworten auf Platz 1", "score": "Punktzahl", "top_5_contributors_today": "Top 5 Benutzer heute", + "total": "Total", "user": "Benutzer", + "username": "Benutzername", "view_all": "Alle anzeigen", - "weekly": "Wöchentlich" + "week": "Woche", + "weekly": "Wöchentlich", + "xp_progress_message": "Du brauchst {{need, number, integer}} mehr Punkte, um das nächste Level zu erreichen!", + "your_account": "Dein Account", + "your_stats": "Deine Statistiken" } diff --git a/website/public/locales/de/message.json b/website/public/locales/de/message.json index b198d6d29..d30d15cbf 100644 --- a/website/public/locales/de/message.json +++ b/website/public/locales/de/message.json @@ -21,6 +21,7 @@ "send_report": "Senden", "stop_tree": "Nachrichtenbaum stoppen", "submit_labels": "Absenden", + "tree_restarted": "Nachrichtenbaum neugestartet {{id}}", "tree_stopped": "Nachrichtenbaum gestoppt {{id}}", "view_user": "Benutzer anzeigen", "your_recent_messages": "Ihre kürzlichen Nachrichten", diff --git a/website/public/locales/de/tasks.json b/website/public/locales/de/tasks.json index ec4e90bf5..e51cf9eb6 100644 --- a/website/public/locales/de/tasks.json +++ b/website/public/locales/de/tasks.json @@ -1,4 +1,5 @@ { + "any_feedback_on_this_task": "Iregndwelches Feedback auf diese Aufgabe?", "available_task_count": "{{count}} Aufgaben verfügbar", "skip_confirmation": "Sind Sie sicher das sie überspringen wollen?", "classify_assistant_reply": { @@ -79,5 +80,11 @@ "overview": "Geben Sie angesichts des folgenden Gesprächs eine angemessene Antwort.", "instruction": "Geben Sie die Antwort des Benutzers an", "response_placeholder": "Schreiben Sie hier Ihre Antwort..." - } + }, + "submitted_as": "This will be submitted as {{submit_lang}}", + "tab_write": "Schreiben", + "tab_preview": "Vorschau", + "writing_wrong_langauge_a_b": "Sie scheinen in {{detected_lang}} zu schreiben, aber dies wird als {{submit_lang}} eingereicht.", + "not_rankable": "Alle Antworten sind sachlich falsch und können nicht gewertet werden", + "moderator_edit_explain": "Ändern Sie die hervorgehobene Nachricht und beschränken Sie die Änderungen auf ein Minimum. Diese Aktion kann dazu führen, dass die Konversation keinen Sinn mehr ergibt und die Bewertungen nicht mehr korrekt sind. Bitte gehen Sie vorsichtig vor." } diff --git a/website/public/locales/en/account.json b/website/public/locales/en/account.json index bf40f7abb..980ffe35e 100644 --- a/website/public/locales/en/account.json +++ b/website/public/locales/en/account.json @@ -4,7 +4,8 @@ "delete_account_intro": "Deleting an account entails the following:", "delete_account_leaderboard": "Accounts of deleted users won't show up on the leaderboards", "delete_account_permanent": "This action is permanent and cannot be undone", - "linked_accounts": "Linked accounts", + "edit_paper_ack": "Edit paper acknowledgement", "go_to_dashboard": "Go back to dashboard", + "linked_accounts": "Linked accounts", "yes_delete": "Yes, delete my account permanently" } diff --git a/website/public/locales/en/chat.json b/website/public/locales/en/chat.json index 413c9f58b..3516b2dd8 100644 --- a/website/public/locales/en/chat.json +++ b/website/public/locales/en/chat.json @@ -33,6 +33,7 @@ "typical_p": "Typical p: Typical sampling is an information-theoretic technique that, in addition to the probability, also considers the sequence entropy (i.e., the information content according to the probability). This means that typical sampling \"overweights\" some of the tokens with lower probability because they are deemed \"interesting,\" and underweights high probability tokens because they are deemed \"boring.\"" }, "plugin_url_placeholder": "Enter plugin URL", + "plugin_repositories": "Plugin Repositories", "plugins": "Plugins", "preset": "Preset", "preset_custom": "Custom", @@ -60,5 +61,10 @@ "preset_name_placeholder": "Enter name", "feedback_message": "How did I do? Your feedback will make me better!", "feedback_action_great": "Good", - "feedback_action_poor": "Could be better" + "feedback_action_poor": "Could be better", + "custom_instructions": "Custom instructions", + "custom_instructions_user_profile": "What info should Open-Assistant have about you to make its replies even better?", + "custom_instructions_response_instructions": "How do you want Open-Assistant to chat with you?", + "custom_instructions_user_profile_placeholder": "List some of your aspirations.\nDescribe your hobbies and interests.\nShare your location.\nWhat is your occupation?\nWhich topics could you discuss extensively?", + "custom_instructions_response_instructions_placeholder": "Should Open-Assistant express opinions or maintain neutrality?\nSpecify the desired formality level for Open-Assistant's responses.\nHow should Open-Assistant address you?\nDetermine the preferred length of responses." } diff --git a/website/public/locales/ru/account.json b/website/public/locales/ru/account.json index af4d73283..173e0c25a 100644 --- a/website/public/locales/ru/account.json +++ b/website/public/locales/ru/account.json @@ -4,6 +4,7 @@ "delete_account_intro": "Удаление аккаунта предполагает следующее:", "delete_account_leaderboard": "Аккаунты удаленных пользователей не будут отображаться в таблице лидеров", "delete_account_permanent": "Это действие является необратимым и не может быть отменено", + "linked_accounts": "Связанные аккаунты", "go_to_dashboard": "Вернуться назад", "yes_delete": "Да, удалить мой аккаунт без возможности восстановления!" } diff --git a/website/public/locales/ru/chat.json b/website/public/locales/ru/chat.json index 3b9bdb6da..963a73151 100644 --- a/website/public/locales/ru/chat.json +++ b/website/public/locales/ru/chat.json @@ -6,13 +6,8 @@ "delete_chat": "Удалить диалог", "delete_confirmation": "Вы уверены, что хотите удалить этот диалог?", "delete_confirmation_detail": "После того, как вы удалите этот чат, его содержание также будет удалено с серверов, и мы не сможем использовать эти данные, чтобы улучшить наши модели. Пожалуйста, уделяйте время тому, чтобы оценивать ответы в других чатах, это поможет нам сделать Open Assistant лучше!", - "opt_out": { - "button": "Отказ от предоставления данных для обучения", - "success_message": "Вы отказались от предоставления данных для обучения.", - "dialog": { - "title": "Подтверждение отказа от предоставления данных для обучения" - } - }, + "draft": "Черновик", + "drafts_generating_notify": "Черновики сообщений все еще генерируются. Пожалуйста, подождите.", "edit_plugin": "Редактировать Плагин", "empty": "(без названия)", "input_placeholder": "Напишите Ассистенту...", @@ -21,7 +16,14 @@ "model": "Модель", "more_actions": "Дополнительные параметры...", "only_visible": "Только Видимые", - "preset": "Предустановка", + "opt_out": { + "button": "Отказ от предоставления данных для обучения", + "success_message": "Вы отказались от предоставления данных для обучения.", + "dialog": { + "title": "Подтверждение отказа от предоставления данных для обучения", + "description": "Если вы подтвердите, то этот чат не будет использован для улучшения нашей модели. Пожалуйста, уделяйте время тому, чтобы оценивать ответы в других чатах, чтобы помочь нам сделать Open Assistant лучше!" + } + }, "parameter_description": { "max_new_tokens": "Максимальное количество новых токенов: Этот параметр указывает модели, сколько максимум новых токенов она должна сгенерировать для ответа.", "repetition_penalty": "Штраф за повторение: Этот параметр уменьшает вероятность повторения одних и тех же токенов снова и снова, делая повторение токенов менее вероятным, чем обычно.", @@ -32,10 +34,12 @@ }, "plugin_url_placeholder": "Введите URL Плагина", "plugins": "Плагины", + "preset": "Предустановка", "preset_custom": "Собственная предустановка", "queue_info": "Ваше сообщение находится в очереди, вы на позиции {{ queuePosition, number, integer }}.", "remove_plugin": "Удалить Плагин", "repetition_penalty": "Штраф за повтор", + "select_chat_notify": "Для продолжения выберите черновик.", "sponsored_by": "При поддержке", "temperature": "Температура", "top_k": "Top K", @@ -56,5 +60,10 @@ "preset_name_placeholder": "Введите имя...", "feedback_message": "Вам понравился мой ответ? Ваш отзыв поможет мне стать лучше!", "feedback_action_great": "Отлично!", - "feedback_action_poor": "Не очень..." + "feedback_action_poor": "Не очень...", + "custom_instructions": "Свои инструкции", + "custom_instructions_user_profile": "Какую информацию Open-Assistant должен иметь о вас, чтобы отвечать ещё лучше?", + "custom_instructions_response_instructions": "Как вы хотите, чтобы Open-Assistant общался с вами?", + "custom_instructions_user_profile_placeholder": "Перечислите свои мечты.\nОпишите свои увлечения и интересы.\nПоделитесь своим местонахождением.\nКакова ваша профессия.\nНа какие темы вы могли бы подробно поговорить?", + "custom_instructions_response_instructions_placeholder": "Должен ли Open-Assistant выражать свое мнение или сохранять нейтралитет?\nУкажите желаемый уровень формальности в ответах Open-Assistant.\nКак Open-Assistant должен обращаться к Вам?\nОпределите предпочтительную длину ответов." } diff --git a/website/public/locales/ru/common.json b/website/public/locales/ru/common.json index 0bb70aecd..a19a6315b 100644 --- a/website/public/locales/ru/common.json +++ b/website/public/locales/ru/common.json @@ -36,6 +36,7 @@ "no": "Нет", "output": "Действие", "parameters": "Показатели", + "plain_text": "Текст без Markdown", "privacy_policy": "Политика конфиденциальности", "prompt": "Запрос", "report_a_bug": "Сообщить об ошибке", @@ -50,6 +51,8 @@ "stats": "Статистика", "status": "Статус", "send": "Отправить", + "show_less": "Показать меньше", + "show_more": "Показать больше", "submit": "Отправить", "submit_your_answer": "Отправить ваш ответ", "success": "Успешно", diff --git a/website/public/locales/ru/message.json b/website/public/locales/ru/message.json index b22460871..494c4d95c 100644 --- a/website/public/locales/ru/message.json +++ b/website/public/locales/ru/message.json @@ -6,11 +6,11 @@ "copy_message_link": "Скопировать ссылку на сообщение", "label_action": "Классифицировать", "label_title": "Классифицировать", - "message_author": "Автор", - "no_messages": "Сообщения отсутствуют", "message_author_explain": "Вы являетесь автором данного сообщения", - "message": "Сообщение", + "message_author": "Автор", "message_deleted": "Сообщение удалено", + "message": "Сообщение", + "no_messages": "Сообщения отсутствуют", "open_new_tab_action": "Открыть в новой вкладке", "parent": "Родитель", "reactions": "Реакции", @@ -18,9 +18,11 @@ "report_action": "Пожаловаться", "report_placeholder": "Почему это сообщение должно быть рассмотрено?", "report_title": "Пожаловаться", + "restart_tree": "Восстановить ветку диалогов", "send_report": "Отправить", "stop_tree": "Завершить ветку диалога", "submit_labels": "Отправить", + "tree_restarted": "Ветка восстановлена {{id}}", "tree_stopped": "Ветка завершена — {{id}}", "view_user": "О пользователе", "your_recent_messages": "Ваши Последние сообщения", diff --git a/website/public/locales/ru/tasks.json b/website/public/locales/ru/tasks.json index bf445bcb2..199ed02b9 100644 --- a/website/public/locales/ru/tasks.json +++ b/website/public/locales/ru/tasks.json @@ -1,6 +1,7 @@ { "any_feedback_on_this_task": "Хотите дать обратную связь по поводу этого задания?", "available_task_count": "Доступных заданий: {{count}}", + "skip_confirmation": "Вы уверены, что хотите пропустить?", "default": { "unchanged_title": "Без изменений", "unchanged_message": "Вы уверены, что хотите продолжить?", @@ -93,5 +94,6 @@ "submitted_as": "Будет помечено как {{submit_lang}} язык", "tab_write": "Редактор (без форматирования)", "tab_preview": "Предпросмотр (с форматированием)", - "not_rankable": "Все предоставленные ответы неверны и не могут быть оценены" + "not_rankable": "Все предоставленные ответы неверны и не могут быть оценены", + "moderator_edit_explain": "Измените выделенное сообщение, сведя изменения к минимуму. Это действие может привести к тому, что беседа потеряет смысл и рейтинги перестанут быть точными. Пожалуйста, используйте его осторожно." } diff --git a/website/public/manifest.json b/website/public/manifest.json new file mode 100644 index 000000000..512dcfe04 --- /dev/null +++ b/website/public/manifest.json @@ -0,0 +1,21 @@ +{ + "name": "Open Assistant", + "short_name": "Open Assistant", + "description": "Open Assistant is a project meant to give everyone access to a great chat based large language model.", + "start_url": "/", + "display": "standalone", + "background_color": "#ffffff", + "theme_color": "#3182CE", + "icons": [ + { + "src": "/images/logos/logo.png", + "sizes": "512x512", + "type": "image/png" + }, + { + "src": "/images/logos/logo_192x192.png", + "sizes": "192x192", + "type": "image/png" + } + ] +} diff --git a/website/src/components/Chat/ChatConfigForm.tsx b/website/src/components/Chat/ChatConfigForm.tsx index 16f16e97b..aad64b053 100644 --- a/website/src/components/Chat/ChatConfigForm.tsx +++ b/website/src/components/Chat/ChatConfigForm.tsx @@ -20,7 +20,13 @@ import { useTranslation } from "next-i18next"; import { ChangeEvent, memo, useCallback, useEffect, useRef, useState } from "react"; import { Controller, useFormContext, UseFormSetValue } from "react-hook-form"; import SimpleBar from "simplebar-react"; -import { ChatConfigFormData, ModelParameterConfig, PluginEntry, SamplingParameters } from "src/types/Chat"; +import { + ChatConfigFormData, + ModelParameterConfig, + PluginEntry, + SamplingParameters, + CustomInstructionsType, +} from "src/types/Chat"; import { CustomPreset, getConfigCache } from "src/utils/chat"; import { useIsomorphicLayoutEffect } from "usehooks-ts"; @@ -28,6 +34,7 @@ import { ChatConfigSaver } from "./ChatConfigSaver"; import { useChatInitialData } from "./ChatInitialDataContext"; import { DeletePresetButton } from "./DeletePresetButton"; import { PluginsChooser } from "./PluginsChooser"; +import CustomInstructions from "./CustomInstructions"; import { SavePresetButton } from "./SavePresetButton"; import { areParametersEqual } from "./WorkParameters"; @@ -104,6 +111,10 @@ export const ChatConfigForm = memo(function ChatConfigForm() { const selectedModel = getValues("model_config_name"); // have to use getValues to here to access latest value const selectedPlugins = getValues("plugins"); const presets = modelInfos.find((model) => model.name === selectedModel)!.parameter_configs; + const [customInstructions, setCustomInstructions] = useState({ + user_profile: "", + user_response_instructions: "", + }); const [selectedPresetName, setSelectedPresetName] = useState(() => findPresetName(presets, getValues())); const { customPresets, handleSavePreset, setCustomPresets, handleDeletePreset } = useCustomPresets({ @@ -114,6 +125,7 @@ export const ChatConfigForm = memo(function ChatConfigForm() { const { hyrated, plugins, setPlugins } = useHydrateChatConfig({ setCustomPresets, setSelectedPresetName, + setCustomInstructions, }); const [lockPresetSelection, setLockPresetSelection] = useState(false); @@ -133,6 +145,14 @@ export const ChatConfigForm = memo(function ChatConfigForm() { [customPresets, presets, setValue] ); + const handleCustomInstructionsChange = useCallback( + (value: CustomInstructionsType) => { + setCustomInstructions(value); + setValue("custom_instructions", value); + }, + [setCustomInstructions] + ); + // Lock preset selection if any plugin is enabled useEffect(() => { const activated = selectedPlugins.some((plugin) => plugin.enabled); @@ -154,6 +174,7 @@ export const ChatConfigForm = memo(function ChatConfigForm() { > + {t("model")}