From f880b3a4f23af1b64f181528c3677b600ccc8b0b Mon Sep 17 00:00:00 2001 From: Paul-Henri Date: Sun, 22 Sep 2024 11:54:32 +0200 Subject: [PATCH 01/17] added a script to restore punctuation --- scripts/restore_punctuation.py | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 scripts/restore_punctuation.py diff --git a/scripts/restore_punctuation.py b/scripts/restore_punctuation.py new file mode 100644 index 00000000..bf665c6b --- /dev/null +++ b/scripts/restore_punctuation.py @@ -0,0 +1,61 @@ +from datasets import load_dataset +from deepmultilingualpunctuation import PunctuationModel +import argparse +from multiprocessing import set_start_method + +def apply_punctuation(examples, model, text_column): + restored_texts = [model.restore_punctuation(text) for text in examples[text_column]] + return { + f"original_{text_column}": examples[text_column], + text_column: restored_texts + } + +if __name__ == "__main__": + set_start_method("spawn") + parser = argparse.ArgumentParser() + + parser.add_argument("dataset_name", type=str, help="Path or name of the dataset. See: https://huggingface.co/docs/datasets/v2.17.0/en/package_reference/loading_methods#datasets.load_dataset.path") + parser.add_argument("--configuration", default=None, type=str, help="Dataset configuration to use, if necessary.") + parser.add_argument("--output_dir", default=None, type=str, help="If specified, save the dataset on disk with this path.") + parser.add_argument("--repo_id", default=None, type=str, help="If specified, push the dataset to the hub.") + parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the text to be punctuated.") + parser.add_argument("--language", default=None, type=str, help="Language of the dataset. If not specified, uses the default multilingual model.") + parser.add_argument("--batch_size", default=32, type=int, help="This parameter specifies how many samples are passed by workers for operations that are using GPUs.") + parser.add_argument("--cpu_num_workers", default=1, type=int, help="Number of CPU workers for transformations that don't use GPUs or if no GPU are available.") + + args = parser.parse_args() + + if args.configuration: + dataset = load_dataset(args.dataset_name, args.configuration, num_proc=args.cpu_num_workers) + else: + dataset = load_dataset(args.dataset_name, num_proc=args.cpu_num_workers) + + if args.language and args.language.lower() not in {"catalan","english", "german", "french", "spanish", "bulgarian", "italian", "polish", "dutch", "czech", "portuguese", "slovak", "slovenian"}: + raise ValueError(f"Language {args.language} is not supported. Please choose from: english, german, french, spanish, bulgarian, italian, polish, dutch, czech, portuguese, slovak, slovenian, catalan") + elif args.language and args.language.lower() == "catalan": + model = PunctuationModel(model=f"softcatala/fullstop-catalan-punctuation-prediction") + elif args.language and args.language.lower() in {"english", "italian", "french", "german", "dutch"}: + model = PunctuationModel(model="oliverguhr/fullstop-punctuation-multilingual-base") + else: + model = PunctuationModel(model=f"kredor/punctuate-all") + + punctuated_dataset = dataset.map( + apply_punctuation, + batched=True, + batch_size=args.batch_size, + fn_kwargs={"model": model, "text_column": args.text_column}, + desc="Restoring punctuation" + ) + + if args.output_dir: + print("Saving to disk...") + punctuated_dataset.save_to_disk(args.output_dir) + + if args.repo_id: + print("Pushing to the hub...") + if args.configuration: + punctuated_dataset.push_to_hub(args.repo_id, args.configuration) + else: + punctuated_dataset.push_to_hub(args.repo_id) + + print("Punctuation restoration completed.") From 3adfe18b97fbc07967e5e886fe00c5f472da15e5 Mon Sep 17 00:00:00 2001 From: Paul-Henri Date: Sun, 22 Sep 2024 11:56:05 +0200 Subject: [PATCH 02/17] updated requirements.txt with deepmultilingualpunctuation --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 93e2b8e4..00ddc667 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ g2p demucs transformers accelerate -bitsandbytes \ No newline at end of file +bitsandbytes +deepmultilingualpunctuation \ No newline at end of file From a3c55017442e553a5561af4cc2446cca71ee88c7 Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 10:58:07 +0200 Subject: [PATCH 03/17] added true casing --- requirements.txt | 3 +- scripts/casing.py | 60 ++++++++++++++ scripts/restore_punctuation.py | 140 ++++++++++++++++++++++++++++----- 3 files changed, 181 insertions(+), 22 deletions(-) create mode 100644 scripts/casing.py diff --git a/requirements.txt b/requirements.txt index 00ddc667..54ccd398 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ demucs transformers accelerate bitsandbytes -deepmultilingualpunctuation \ No newline at end of file +deepmultilingualpunctuation +spacy \ No newline at end of file diff --git a/scripts/casing.py b/scripts/casing.py new file mode 100644 index 00000000..e4416257 --- /dev/null +++ b/scripts/casing.py @@ -0,0 +1,60 @@ +from datasets import load_dataset +import argparse +import re + +def capitalize_first_letter(text): + return '. '.join(sentence.capitalize() for sentence in text.split('. ')) + +def capitalize_words_remove_quotes(text): + # Remove quotes and capitalize each word + return ' '.join(word.capitalize() for word in re.findall(r'\w+', text)) + +def apply_recasing(examples, text_column, description_column): + recased_texts = [capitalize_first_letter(text) for text in examples[text_column]] + recased_descriptions = [capitalize_words_remove_quotes(desc) for desc in examples[description_column]] + return { + f"original_{text_column}": examples[text_column], + text_column: recased_texts, + f"original_{description_column}": examples[description_column], + description_column: recased_descriptions + } + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("dataset_name", type=str, help="Path or name of the dataset.") + parser.add_argument("--configuration", default=None, type=str, help="Dataset configuration to use, if necessary.") + parser.add_argument("--output_dir", default=None, type=str, help="If specified, save the dataset on disk with this path.") + parser.add_argument("--repo_id", default=None, type=str, help="If specified, push the dataset to the hub.") + parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the text to be recased.") + parser.add_argument("--description_column", default="text_description", type=str, help="Name of the column containing the description to be recased and cleaned.") + parser.add_argument("--batch_size", default=32, type=int, help="Batch size for processing.") + parser.add_argument("--num_proc", default=1, type=int, help="Number of processes to use.") + + args = parser.parse_args() + + if args.configuration: + dataset = load_dataset(args.dataset_name, args.configuration, num_proc=args.num_proc) + else: + dataset = load_dataset(args.dataset_name, num_proc=args.num_proc) + + recased_dataset = dataset.map( + apply_recasing, + batched=True, + batch_size=args.batch_size, + fn_kwargs={"text_column": args.text_column, "description_column": args.description_column}, + desc="Applying recasing" + ) + + if args.output_dir: + print("Saving to disk...") + recased_dataset.save_to_disk(args.output_dir) + + if args.repo_id: + print("Pushing to the hub...") + if args.configuration: + recased_dataset.push_to_hub(args.repo_id, args.configuration) + else: + recased_dataset.push_to_hub(args.repo_id) + + print("Recasing completed.") \ No newline at end of file diff --git a/scripts/restore_punctuation.py b/scripts/restore_punctuation.py index bf665c6b..91a7493b 100644 --- a/scripts/restore_punctuation.py +++ b/scripts/restore_punctuation.py @@ -1,13 +1,91 @@ -from datasets import load_dataset -from deepmultilingualpunctuation import PunctuationModel import argparse from multiprocessing import set_start_method +from datasets import load_dataset +from deepmultilingualpunctuation import PunctuationModel +import spacy +from typing import Dict, Callable + +nlp_models: Dict[str, spacy.language.Language] = {} + +def load_spacy_model(lang_code: str) -> spacy.language.Language: + """Load and return the appropriate spaCy model for the given language code.""" + if lang_code not in nlp_models: + model_name = { + 'ca': 'ca_core_news_sm', + 'en': 'en_core_web_sm', + 'de': 'de_core_news_sm', + 'fr': 'fr_core_news_sm', + 'es': 'es_core_news_sm', + 'bg': 'bg_core_news_sm', + 'it': 'it_core_news_sm', + 'pl': 'pl_core_news_sm', + 'nl': 'nl_core_news_sm', + 'cs': 'cs_core_news_sm', + 'pt': 'pt_core_news_sm', + 'sk': 'sk_core_news_sm', + 'sl': 'sl_core_news_sm' + }.get(lang_code) + + if model_name is None: + raise ValueError(f"Unsupported language code: {lang_code}") + + nlp_models[lang_code] = spacy.load(model_name) + + return nlp_models[lang_code] + +def get_capitalization_function(lang_code: str) -> Callable[[spacy.tokens.Token], str]: + """Return the appropriate capitalization function for the given language.""" + + def default_capitalization(token: spacy.tokens.Token) -> str: + if token.is_sent_start or token.pos_ in ('PROPN', 'NNP', 'NNPS'): + return token.text.capitalize() + return token.text.lower() + + def german_capitalization(token: spacy.tokens.Token) -> str: + if token.is_sent_start or token.pos_ in ('PROPN', 'NOUN'): + return token.text.capitalize() + return token.text.lower() + + if lang_code == 'de': + return german_capitalization + else: + return default_capitalization -def apply_punctuation(examples, model, text_column): - restored_texts = [model.restore_punctuation(text) for text in examples[text_column]] +def true_case(text: str, lang_code: str) -> str: + """ + Perform true casing on the input text for the specified language. + + :param text: Input text to be true cased + :param lang_code: Two-letter language code (e.g., 'en' for English) + :return: True cased text + """ + nlp = load_spacy_model(lang_code) + capitalization_func = get_capitalization_function(lang_code) + + doc = nlp(text) + true_cased_tokens = [capitalization_func(token) for token in doc] + + # Join tokens, ensuring no space before punctuation + true_cased_text = "" + for i, token in enumerate(doc): + if i > 0 and not token.is_punct: + true_cased_text += " " + true_cased_text += true_cased_tokens[i] + + return true_cased_text + +def apply_processing(examples, punctuation_model, text_column, lang_code, punctuation_only, truecase_only): + if punctuation_only: + processed_texts = [punctuation_model.restore_punctuation(text) for text in examples[text_column]] + elif truecase_only: + processed_texts = [true_case(text, lang_code) for text in examples[text_column]] + else: + restored_texts = [punctuation_model.restore_punctuation(text) for text in examples[text_column]] + processed_texts = [true_case(text, lang_code) for text in restored_texts] + return { f"original_{text_column}": examples[text_column], - text_column: restored_texts + text_column: processed_texts } if __name__ == "__main__": @@ -18,44 +96,64 @@ def apply_punctuation(examples, model, text_column): parser.add_argument("--configuration", default=None, type=str, help="Dataset configuration to use, if necessary.") parser.add_argument("--output_dir", default=None, type=str, help="If specified, save the dataset on disk with this path.") parser.add_argument("--repo_id", default=None, type=str, help="If specified, push the dataset to the hub.") - parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the text to be punctuated.") + parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the text to be processed.") parser.add_argument("--language", default=None, type=str, help="Language of the dataset. If not specified, uses the default multilingual model.") parser.add_argument("--batch_size", default=32, type=int, help="This parameter specifies how many samples are passed by workers for operations that are using GPUs.") parser.add_argument("--cpu_num_workers", default=1, type=int, help="Number of CPU workers for transformations that don't use GPUs or if no GPU are available.") + parser.add_argument("--punctuation_only", action="store_true", help="If set, only perform punctuation restoration.") + parser.add_argument("--truecase_only", action="store_true", help="If set, only perform true casing.") args = parser.parse_args() + if args.punctuation_only and args.truecase_only: + raise ValueError("Cannot set both --punctuation_only and --truecase_only. Choose one or neither.") + if args.configuration: dataset = load_dataset(args.dataset_name, args.configuration, num_proc=args.cpu_num_workers) else: dataset = load_dataset(args.dataset_name, num_proc=args.cpu_num_workers) - if args.language and args.language.lower() not in {"catalan","english", "german", "french", "spanish", "bulgarian", "italian", "polish", "dutch", "czech", "portuguese", "slovak", "slovenian"}: - raise ValueError(f"Language {args.language} is not supported. Please choose from: english, german, french, spanish, bulgarian, italian, polish, dutch, czech, portuguese, slovak, slovenian, catalan") - elif args.language and args.language.lower() == "catalan": - model = PunctuationModel(model=f"softcatala/fullstop-catalan-punctuation-prediction") - elif args.language and args.language.lower() in {"english", "italian", "french", "german", "dutch"}: - model = PunctuationModel(model="oliverguhr/fullstop-punctuation-multilingual-base") + supported_languages = {"catalan", "english", "german", "french", "spanish", "bulgarian", "italian", "polish", "dutch", "czech", "portuguese", "slovak", "slovenian"} + if args.language and args.language.lower() not in supported_languages: + raise ValueError(f"Language {args.language} is not supported. Please choose from: {', '.join(supported_languages)}") + + lang_code = args.language.lower()[:2] if args.language else 'en' # Default to English if no language is specified + + if lang_code == "ca": + punctuation_model = PunctuationModel(model="softcatala/fullstop-catalan-punctuation-prediction") + elif lang_code in {"en", "it", "fr", "de", "nl"}: + punctuation_model = PunctuationModel(model="oliverguhr/fullstop-punctuation-multilingual-base") else: - model = PunctuationModel(model=f"kredor/punctuate-all") + punctuation_model = PunctuationModel(model="kredor/punctuate-all") - punctuated_dataset = dataset.map( - apply_punctuation, + processed_dataset = dataset.map( + apply_processing, batched=True, batch_size=args.batch_size, - fn_kwargs={"model": model, "text_column": args.text_column}, - desc="Restoring punctuation" + fn_kwargs={ + "punctuation_model": punctuation_model, + "text_column": args.text_column, + "lang_code": lang_code, + "punctuation_only": args.punctuation_only, + "truecase_only": args.truecase_only + }, + desc="Processing text" ) if args.output_dir: print("Saving to disk...") - punctuated_dataset.save_to_disk(args.output_dir) + processed_dataset.save_to_disk(args.output_dir) if args.repo_id: print("Pushing to the hub...") if args.configuration: - punctuated_dataset.push_to_hub(args.repo_id, args.configuration) + processed_dataset.push_to_hub(args.repo_id, args.configuration) else: - punctuated_dataset.push_to_hub(args.repo_id) + processed_dataset.push_to_hub(args.repo_id) - print("Punctuation restoration completed.") + if args.punctuation_only: + print("Punctuation restoration completed.") + elif args.truecase_only: + print("True casing completed.") + else: + print("Punctuation restoration and true casing completed.") From d85bd6d51de1c9cb0e876f457e61d94434f5c7a3 Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 11:46:57 +0200 Subject: [PATCH 04/17] updated handling of language --- scripts/restore_punctuation.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/restore_punctuation.py b/scripts/restore_punctuation.py index 91a7493b..9283d844 100644 --- a/scripts/restore_punctuation.py +++ b/scripts/restore_punctuation.py @@ -113,11 +113,27 @@ def apply_processing(examples, punctuation_model, text_column, lang_code, punctu else: dataset = load_dataset(args.dataset_name, num_proc=args.cpu_num_workers) - supported_languages = {"catalan", "english", "german", "french", "spanish", "bulgarian", "italian", "polish", "dutch", "czech", "portuguese", "slovak", "slovenian"} + language_to_code = { + "catalan": "ca", + "english": "en", + "german": "de", + "french": "fr", + "spanish": "es", + "bulgarian": "bg", + "italian": "it", + "polish": "pl", + "dutch": "nl", + "czech": "cs", + "portuguese": "pt", + "slovak": "sk", + "slovenian": "sl" + } + + supported_languages = set(language_to_code.keys()) if args.language and args.language.lower() not in supported_languages: raise ValueError(f"Language {args.language} is not supported. Please choose from: {', '.join(supported_languages)}") - lang_code = args.language.lower()[:2] if args.language else 'en' # Default to English if no language is specified + lang_code = language_to_code[args.language.lower()] if args.language else 'en' if lang_code == "ca": punctuation_model = PunctuationModel(model="softcatala/fullstop-catalan-punctuation-prediction") From d25b7d203d85094a2a51298fea3e2f6f35e103ad Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 11:49:15 +0200 Subject: [PATCH 05/17] updated requirements.txt --- requirements.txt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 54ccd398..1e0514e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,17 @@ transformers accelerate bitsandbytes deepmultilingualpunctuation -spacy \ No newline at end of file +spacy +https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.5.0/de_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.5.0/fr_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.5.0/es_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/it_core_news_sm-3.5.0/it_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.5.0/nl_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/pl_core_news_sm-3.5.0/pl_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.5.0/pt_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.5.0/ca_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/bg_core_news_sm-3.5.0/bg_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/cs_core_news_sm-3.5.0/cs_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/sk_core_news_sm-3.5.0/sk_core_news_sm-3.5.0-py3-none-any.whl +https://github.com/explosion/spacy-models/releases/download/sl_core_news_sm-3.5.0/sl_core_news_sm-3.5.0-py3-none-any.whl \ No newline at end of file From 9b7ddb9b200535f11c427da6f7403f2ca758cd1c Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 11:58:52 +0200 Subject: [PATCH 06/17] removed unecessary languages --- requirements.txt | 6 +----- scripts/restore_punctuation.py | 8 -------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1e0514e4..773beffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,8 +16,4 @@ https://github.com/explosion/spacy-models/releases/download/it_core_news_sm-3.5. https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.5.0/nl_core_news_sm-3.5.0-py3-none-any.whl https://github.com/explosion/spacy-models/releases/download/pl_core_news_sm-3.5.0/pl_core_news_sm-3.5.0-py3-none-any.whl https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.5.0/pt_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.5.0/ca_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/bg_core_news_sm-3.5.0/bg_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/cs_core_news_sm-3.5.0/cs_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/sk_core_news_sm-3.5.0/sk_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/sl_core_news_sm-3.5.0/sl_core_news_sm-3.5.0-py3-none-any.whl \ No newline at end of file +https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.5.0/ca_core_news_sm-3.5.0-py3-none-any.whl \ No newline at end of file diff --git a/scripts/restore_punctuation.py b/scripts/restore_punctuation.py index 9283d844..5efcc278 100644 --- a/scripts/restore_punctuation.py +++ b/scripts/restore_punctuation.py @@ -16,14 +16,10 @@ def load_spacy_model(lang_code: str) -> spacy.language.Language: 'de': 'de_core_news_sm', 'fr': 'fr_core_news_sm', 'es': 'es_core_news_sm', - 'bg': 'bg_core_news_sm', 'it': 'it_core_news_sm', 'pl': 'pl_core_news_sm', 'nl': 'nl_core_news_sm', - 'cs': 'cs_core_news_sm', 'pt': 'pt_core_news_sm', - 'sk': 'sk_core_news_sm', - 'sl': 'sl_core_news_sm' }.get(lang_code) if model_name is None: @@ -119,14 +115,10 @@ def apply_processing(examples, punctuation_model, text_column, lang_code, punctu "german": "de", "french": "fr", "spanish": "es", - "bulgarian": "bg", "italian": "it", "polish": "pl", "dutch": "nl", - "czech": "cs", "portuguese": "pt", - "slovak": "sk", - "slovenian": "sl" } supported_languages = set(language_to_code.keys()) From 289212cbd24c90d5e0c57af62cfe8f731945072c Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 12:06:15 +0200 Subject: [PATCH 07/17] requirements test --- requirements.txt | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 773beffe..0919f2d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,12 +8,5 @@ accelerate bitsandbytes deepmultilingualpunctuation spacy -https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.5.0/de_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.5.0/fr_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.5.0/es_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/it_core_news_sm-3.5.0/it_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.5.0/nl_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/pl_core_news_sm-3.5.0/pl_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.5.0/pt_core_news_sm-3.5.0-py3-none-any.whl -https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.5.0/ca_core_news_sm-3.5.0-py3-none-any.whl \ No newline at end of file +en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl +en_core_web_ca @ https://github.com/explosion/spacy-models/releases/download/ca_core_web_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl From b21b27caa6d7cbca0df8afcb9fe05ae52e65a149 Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 12:06:58 +0200 Subject: [PATCH 08/17] requirements test --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0919f2d9..61039806 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ bitsandbytes deepmultilingualpunctuation spacy en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl -en_core_web_ca @ https://github.com/explosion/spacy-models/releases/download/ca_core_web_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl +ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_web_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl From c8c447c832fb391790249f56d840d1714026c10f Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 12:16:39 +0200 Subject: [PATCH 09/17] update requirements --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 61039806..45668ba6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,6 @@ accelerate bitsandbytes deepmultilingualpunctuation spacy -en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl -ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_web_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl +en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_news_sm-3.7.1/en_core_news_sm-3.7.1-py3-none-any.whl +ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl +nl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_nesw_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file From ba519c84f29d6331687280c94ea237ec455c05ef Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 12:17:21 +0200 Subject: [PATCH 10/17] update requirements --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 45668ba6..0830a32d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ accelerate bitsandbytes deepmultilingualpunctuation spacy -en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_news_sm-3.7.1/en_core_news_sm-3.7.1-py3-none-any.whl +en_core_news_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_news_sm-3.7.1/en_core_news_sm-3.7.1-py3-none-any.whl ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl -nl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_nesw_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file +nl-core-nw-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_nesw_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file From 13e314cc1708811d827ae43928610d4288c921ee Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 12:18:03 +0200 Subject: [PATCH 11/17] update requirements --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0830a32d..1bff65f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ accelerate bitsandbytes deepmultilingualpunctuation spacy -en_core_news_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_news_sm-3.7.1/en_core_news_sm-3.7.1-py3-none-any.whl -ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl -nl-core-nw-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_nesw_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file +en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl +ca-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_web_sm-3.8.0/ca_core_web_sm-3.8.0-py3-none-any.whl +nl-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_web_sm-3.8.0/nl_core_web_sm-3.8.0-py3-none-any.whl \ No newline at end of file From a1a0e79a31485a31dcab357d1a04395b7d61c680 Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 12:19:30 +0200 Subject: [PATCH 12/17] update requirements --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1bff65f2..3b8fd1d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,5 @@ bitsandbytes deepmultilingualpunctuation spacy en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl -ca-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_web_sm-3.8.0/ca_core_web_sm-3.8.0-py3-none-any.whl -nl-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_web_sm-3.8.0/nl_core_web_sm-3.8.0-py3-none-any.whl \ No newline at end of file +ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl +nl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file From e77e9e00ab855aff5cad75440dd5d54b4014b14f Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 12:21:20 +0200 Subject: [PATCH 13/17] update requirements --- requirements.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3b8fd1d3..df38c1c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,12 @@ accelerate bitsandbytes deepmultilingualpunctuation spacy -en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl -nl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl +de-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.8.0/de_core_news_sm-3.8.0-py3-none-any.whl +fr-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.8.0/fr_core_news_sm-3.8.0-py3-none-any.whl +es-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.8.0/es_core_news_sm-3.8.0-py3-none-any.whl +it-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/it_core_news_sm-3.8.0/it_core_news_sm-3.8.0-py3-none-any.whl +pl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/pl_core_news_sm-3.8.0/pl_core_news_sm-3.8.0-py3-none-any.whl +nl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl +pt-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.8.0/pt_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file From 0d8ed8538d06f870603be1a7471db3581d20b052 Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 10:58:07 +0200 Subject: [PATCH 14/17] Added true casing for 8 languages --- requirements.txt | 12 ++- scripts/casing.py | 60 +++++++++++++ scripts/restore_punctuation.py | 148 ++++++++++++++++++++++++++++----- 3 files changed, 198 insertions(+), 22 deletions(-) create mode 100644 scripts/casing.py diff --git a/requirements.txt b/requirements.txt index 00ddc667..df38c1c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,14 @@ demucs transformers accelerate bitsandbytes -deepmultilingualpunctuation \ No newline at end of file +deepmultilingualpunctuation +spacy +ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl +de-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.8.0/de_core_news_sm-3.8.0-py3-none-any.whl +fr-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.8.0/fr_core_news_sm-3.8.0-py3-none-any.whl +es-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.8.0/es_core_news_sm-3.8.0-py3-none-any.whl +it-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/it_core_news_sm-3.8.0/it_core_news_sm-3.8.0-py3-none-any.whl +pl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/pl_core_news_sm-3.8.0/pl_core_news_sm-3.8.0-py3-none-any.whl +nl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl +pt-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.8.0/pt_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file diff --git a/scripts/casing.py b/scripts/casing.py new file mode 100644 index 00000000..e4416257 --- /dev/null +++ b/scripts/casing.py @@ -0,0 +1,60 @@ +from datasets import load_dataset +import argparse +import re + +def capitalize_first_letter(text): + return '. '.join(sentence.capitalize() for sentence in text.split('. ')) + +def capitalize_words_remove_quotes(text): + # Remove quotes and capitalize each word + return ' '.join(word.capitalize() for word in re.findall(r'\w+', text)) + +def apply_recasing(examples, text_column, description_column): + recased_texts = [capitalize_first_letter(text) for text in examples[text_column]] + recased_descriptions = [capitalize_words_remove_quotes(desc) for desc in examples[description_column]] + return { + f"original_{text_column}": examples[text_column], + text_column: recased_texts, + f"original_{description_column}": examples[description_column], + description_column: recased_descriptions + } + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("dataset_name", type=str, help="Path or name of the dataset.") + parser.add_argument("--configuration", default=None, type=str, help="Dataset configuration to use, if necessary.") + parser.add_argument("--output_dir", default=None, type=str, help="If specified, save the dataset on disk with this path.") + parser.add_argument("--repo_id", default=None, type=str, help="If specified, push the dataset to the hub.") + parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the text to be recased.") + parser.add_argument("--description_column", default="text_description", type=str, help="Name of the column containing the description to be recased and cleaned.") + parser.add_argument("--batch_size", default=32, type=int, help="Batch size for processing.") + parser.add_argument("--num_proc", default=1, type=int, help="Number of processes to use.") + + args = parser.parse_args() + + if args.configuration: + dataset = load_dataset(args.dataset_name, args.configuration, num_proc=args.num_proc) + else: + dataset = load_dataset(args.dataset_name, num_proc=args.num_proc) + + recased_dataset = dataset.map( + apply_recasing, + batched=True, + batch_size=args.batch_size, + fn_kwargs={"text_column": args.text_column, "description_column": args.description_column}, + desc="Applying recasing" + ) + + if args.output_dir: + print("Saving to disk...") + recased_dataset.save_to_disk(args.output_dir) + + if args.repo_id: + print("Pushing to the hub...") + if args.configuration: + recased_dataset.push_to_hub(args.repo_id, args.configuration) + else: + recased_dataset.push_to_hub(args.repo_id) + + print("Recasing completed.") \ No newline at end of file diff --git a/scripts/restore_punctuation.py b/scripts/restore_punctuation.py index bf665c6b..5efcc278 100644 --- a/scripts/restore_punctuation.py +++ b/scripts/restore_punctuation.py @@ -1,13 +1,87 @@ -from datasets import load_dataset -from deepmultilingualpunctuation import PunctuationModel import argparse from multiprocessing import set_start_method +from datasets import load_dataset +from deepmultilingualpunctuation import PunctuationModel +import spacy +from typing import Dict, Callable + +nlp_models: Dict[str, spacy.language.Language] = {} + +def load_spacy_model(lang_code: str) -> spacy.language.Language: + """Load and return the appropriate spaCy model for the given language code.""" + if lang_code not in nlp_models: + model_name = { + 'ca': 'ca_core_news_sm', + 'en': 'en_core_web_sm', + 'de': 'de_core_news_sm', + 'fr': 'fr_core_news_sm', + 'es': 'es_core_news_sm', + 'it': 'it_core_news_sm', + 'pl': 'pl_core_news_sm', + 'nl': 'nl_core_news_sm', + 'pt': 'pt_core_news_sm', + }.get(lang_code) + + if model_name is None: + raise ValueError(f"Unsupported language code: {lang_code}") + + nlp_models[lang_code] = spacy.load(model_name) + + return nlp_models[lang_code] -def apply_punctuation(examples, model, text_column): - restored_texts = [model.restore_punctuation(text) for text in examples[text_column]] +def get_capitalization_function(lang_code: str) -> Callable[[spacy.tokens.Token], str]: + """Return the appropriate capitalization function for the given language.""" + + def default_capitalization(token: spacy.tokens.Token) -> str: + if token.is_sent_start or token.pos_ in ('PROPN', 'NNP', 'NNPS'): + return token.text.capitalize() + return token.text.lower() + + def german_capitalization(token: spacy.tokens.Token) -> str: + if token.is_sent_start or token.pos_ in ('PROPN', 'NOUN'): + return token.text.capitalize() + return token.text.lower() + + if lang_code == 'de': + return german_capitalization + else: + return default_capitalization + +def true_case(text: str, lang_code: str) -> str: + """ + Perform true casing on the input text for the specified language. + + :param text: Input text to be true cased + :param lang_code: Two-letter language code (e.g., 'en' for English) + :return: True cased text + """ + nlp = load_spacy_model(lang_code) + capitalization_func = get_capitalization_function(lang_code) + + doc = nlp(text) + true_cased_tokens = [capitalization_func(token) for token in doc] + + # Join tokens, ensuring no space before punctuation + true_cased_text = "" + for i, token in enumerate(doc): + if i > 0 and not token.is_punct: + true_cased_text += " " + true_cased_text += true_cased_tokens[i] + + return true_cased_text + +def apply_processing(examples, punctuation_model, text_column, lang_code, punctuation_only, truecase_only): + if punctuation_only: + processed_texts = [punctuation_model.restore_punctuation(text) for text in examples[text_column]] + elif truecase_only: + processed_texts = [true_case(text, lang_code) for text in examples[text_column]] + else: + restored_texts = [punctuation_model.restore_punctuation(text) for text in examples[text_column]] + processed_texts = [true_case(text, lang_code) for text in restored_texts] + return { f"original_{text_column}": examples[text_column], - text_column: restored_texts + text_column: processed_texts } if __name__ == "__main__": @@ -18,44 +92,76 @@ def apply_punctuation(examples, model, text_column): parser.add_argument("--configuration", default=None, type=str, help="Dataset configuration to use, if necessary.") parser.add_argument("--output_dir", default=None, type=str, help="If specified, save the dataset on disk with this path.") parser.add_argument("--repo_id", default=None, type=str, help="If specified, push the dataset to the hub.") - parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the text to be punctuated.") + parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the text to be processed.") parser.add_argument("--language", default=None, type=str, help="Language of the dataset. If not specified, uses the default multilingual model.") parser.add_argument("--batch_size", default=32, type=int, help="This parameter specifies how many samples are passed by workers for operations that are using GPUs.") parser.add_argument("--cpu_num_workers", default=1, type=int, help="Number of CPU workers for transformations that don't use GPUs or if no GPU are available.") + parser.add_argument("--punctuation_only", action="store_true", help="If set, only perform punctuation restoration.") + parser.add_argument("--truecase_only", action="store_true", help="If set, only perform true casing.") args = parser.parse_args() + if args.punctuation_only and args.truecase_only: + raise ValueError("Cannot set both --punctuation_only and --truecase_only. Choose one or neither.") + if args.configuration: dataset = load_dataset(args.dataset_name, args.configuration, num_proc=args.cpu_num_workers) else: dataset = load_dataset(args.dataset_name, num_proc=args.cpu_num_workers) - if args.language and args.language.lower() not in {"catalan","english", "german", "french", "spanish", "bulgarian", "italian", "polish", "dutch", "czech", "portuguese", "slovak", "slovenian"}: - raise ValueError(f"Language {args.language} is not supported. Please choose from: english, german, french, spanish, bulgarian, italian, polish, dutch, czech, portuguese, slovak, slovenian, catalan") - elif args.language and args.language.lower() == "catalan": - model = PunctuationModel(model=f"softcatala/fullstop-catalan-punctuation-prediction") - elif args.language and args.language.lower() in {"english", "italian", "french", "german", "dutch"}: - model = PunctuationModel(model="oliverguhr/fullstop-punctuation-multilingual-base") + language_to_code = { + "catalan": "ca", + "english": "en", + "german": "de", + "french": "fr", + "spanish": "es", + "italian": "it", + "polish": "pl", + "dutch": "nl", + "portuguese": "pt", + } + + supported_languages = set(language_to_code.keys()) + if args.language and args.language.lower() not in supported_languages: + raise ValueError(f"Language {args.language} is not supported. Please choose from: {', '.join(supported_languages)}") + + lang_code = language_to_code[args.language.lower()] if args.language else 'en' + + if lang_code == "ca": + punctuation_model = PunctuationModel(model="softcatala/fullstop-catalan-punctuation-prediction") + elif lang_code in {"en", "it", "fr", "de", "nl"}: + punctuation_model = PunctuationModel(model="oliverguhr/fullstop-punctuation-multilingual-base") else: - model = PunctuationModel(model=f"kredor/punctuate-all") + punctuation_model = PunctuationModel(model="kredor/punctuate-all") - punctuated_dataset = dataset.map( - apply_punctuation, + processed_dataset = dataset.map( + apply_processing, batched=True, batch_size=args.batch_size, - fn_kwargs={"model": model, "text_column": args.text_column}, - desc="Restoring punctuation" + fn_kwargs={ + "punctuation_model": punctuation_model, + "text_column": args.text_column, + "lang_code": lang_code, + "punctuation_only": args.punctuation_only, + "truecase_only": args.truecase_only + }, + desc="Processing text" ) if args.output_dir: print("Saving to disk...") - punctuated_dataset.save_to_disk(args.output_dir) + processed_dataset.save_to_disk(args.output_dir) if args.repo_id: print("Pushing to the hub...") if args.configuration: - punctuated_dataset.push_to_hub(args.repo_id, args.configuration) + processed_dataset.push_to_hub(args.repo_id, args.configuration) else: - punctuated_dataset.push_to_hub(args.repo_id) + processed_dataset.push_to_hub(args.repo_id) - print("Punctuation restoration completed.") + if args.punctuation_only: + print("Punctuation restoration completed.") + elif args.truecase_only: + print("True casing completed.") + else: + print("Punctuation restoration and true casing completed.") From 018a3d3368d4bacfc91c3dd5fca86a139473b5c0 Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Wed, 2 Oct 2024 13:56:02 +0200 Subject: [PATCH 15/17] added processing of the text_description column --- scripts/restore_punctuation.py | 74 +++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/scripts/restore_punctuation.py b/scripts/restore_punctuation.py index 5efcc278..3f644d51 100644 --- a/scripts/restore_punctuation.py +++ b/scripts/restore_punctuation.py @@ -3,7 +3,8 @@ from datasets import load_dataset from deepmultilingualpunctuation import PunctuationModel import spacy -from typing import Dict, Callable +from typing import Dict, Callable, List +import re nlp_models: Dict[str, spacy.language.Language] = {} @@ -70,40 +71,67 @@ def true_case(text: str, lang_code: str) -> str: return true_cased_text -def apply_processing(examples, punctuation_model, text_column, lang_code, punctuation_only, truecase_only): - if punctuation_only: - processed_texts = [punctuation_model.restore_punctuation(text) for text in examples[text_column]] - elif truecase_only: - processed_texts = [true_case(text, lang_code) for text in examples[text_column]] - else: - restored_texts = [punctuation_model.restore_punctuation(text) for text in examples[text_column]] - processed_texts = [true_case(text, lang_code) for text in restored_texts] +def remove_quotes(text: str) -> str: + """Remove single and double quotes from the input text.""" + return re.sub(r"['\"]", "", text) + +def apply_processing(examples, punctuation_model, text_column, description_column, lang_code, punctuation_only, truecase_only): + result = {} - return { - f"original_{text_column}": examples[text_column], - text_column: processed_texts - } + # Process text column + if text_column: + if punctuation_only: + processed_texts = [punctuation_model.restore_punctuation(text) for text in examples[text_column]] + elif truecase_only: + processed_texts = [true_case(text, lang_code) for text in examples[text_column]] + else: + restored_texts = [punctuation_model.restore_punctuation(text) for text in examples[text_column]] + processed_texts = [true_case(text, lang_code) for text in restored_texts] + + result[f"original_{text_column}"] = examples[text_column] + result[text_column] = processed_texts + + # Process description column + if description_column: + if punctuation_only: + processed_descriptions = [remove_quotes(punctuation_model.restore_punctuation(text)) for text in examples[description_column]] + elif truecase_only: + processed_descriptions = [remove_quotes(true_case(text, lang_code)) for text in examples[description_column]] + else: + restored_descriptions = [punctuation_model.restore_punctuation(text) for text in examples[description_column]] + processed_descriptions = [remove_quotes(true_case(text, lang_code)) for text in restored_descriptions] + + result[f"original_{description_column}"] = examples[description_column] + result[description_column] = processed_descriptions + + return result if __name__ == "__main__": set_start_method("spawn") parser = argparse.ArgumentParser() - parser.add_argument("dataset_name", type=str, help="Path or name of the dataset. See: https://huggingface.co/docs/datasets/v2.17.0/en/package_reference/loading_methods#datasets.load_dataset.path") + parser.add_argument("dataset_name", type=str, help="Path or name of the dataset.") parser.add_argument("--configuration", default=None, type=str, help="Dataset configuration to use, if necessary.") parser.add_argument("--output_dir", default=None, type=str, help="If specified, save the dataset on disk with this path.") parser.add_argument("--repo_id", default=None, type=str, help="If specified, push the dataset to the hub.") - parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the text to be processed.") + parser.add_argument("--text_column", default="text", type=str, help="Name of the column containing the main text to be processed.") + parser.add_argument("--description_column", default="text_description", type=str, help="Name of the column containing the description text to be processed.") parser.add_argument("--language", default=None, type=str, help="Language of the dataset. If not specified, uses the default multilingual model.") parser.add_argument("--batch_size", default=32, type=int, help="This parameter specifies how many samples are passed by workers for operations that are using GPUs.") parser.add_argument("--cpu_num_workers", default=1, type=int, help="Number of CPU workers for transformations that don't use GPUs or if no GPU are available.") parser.add_argument("--punctuation_only", action="store_true", help="If set, only perform punctuation restoration.") parser.add_argument("--truecase_only", action="store_true", help="If set, only perform true casing.") + parser.add_argument("--process_text_only", action="store_true", help="If set, only process the text column.") + parser.add_argument("--process_description_only", action="store_true", help="If set, only process the description column.") args = parser.parse_args() if args.punctuation_only and args.truecase_only: raise ValueError("Cannot set both --punctuation_only and --truecase_only. Choose one or neither.") + if args.process_text_only and args.process_description_only: + raise ValueError("Cannot set both --process_text_only and --process_description_only. Choose one or neither.") + if args.configuration: dataset = load_dataset(args.dataset_name, args.configuration, num_proc=args.cpu_num_workers) else: @@ -134,13 +162,18 @@ def apply_processing(examples, punctuation_model, text_column, lang_code, punctu else: punctuation_model = PunctuationModel(model="kredor/punctuate-all") + # Determine which columns to process + text_column = args.text_column if not args.process_description_only else None + description_column = args.description_column if not args.process_text_only else None + processed_dataset = dataset.map( apply_processing, batched=True, batch_size=args.batch_size, fn_kwargs={ "punctuation_model": punctuation_model, - "text_column": args.text_column, + "text_column": text_column, + "description_column": description_column, "lang_code": lang_code, "punctuation_only": args.punctuation_only, "truecase_only": args.truecase_only @@ -159,9 +192,12 @@ def apply_processing(examples, punctuation_model, text_column, lang_code, punctu else: processed_dataset.push_to_hub(args.repo_id) + print("Processing completed for the following columns:", ", ".join(filter(None, [text_column, description_column]))) if args.punctuation_only: - print("Punctuation restoration completed.") + print("Operation: Punctuation restoration") elif args.truecase_only: - print("True casing completed.") + print("Operation: True casing") else: - print("Punctuation restoration and true casing completed.") + print("Operation: Punctuation restoration and true casing") + if description_column: + print("Note: Single and double quotes were removed from the description column") \ No newline at end of file From e3bf102b4933e7c2e542cee4ac7a8f5d9f43b4ab Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Thu, 31 Oct 2024 12:37:12 +0100 Subject: [PATCH 16/17] moved the dowload of the spacy models inside the restore punctuation script --- requirements.txt | 11 +---------- scripts/restore_punctuation.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index df38c1c0..54ccd398 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,13 +7,4 @@ transformers accelerate bitsandbytes deepmultilingualpunctuation -spacy -ca-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ca_core_news_sm-3.8.0/ca_core_news_sm-3.8.0-py3-none-any.whl -en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl -de-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.8.0/de_core_news_sm-3.8.0-py3-none-any.whl -fr-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.8.0/fr_core_news_sm-3.8.0-py3-none-any.whl -es-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.8.0/es_core_news_sm-3.8.0-py3-none-any.whl -it-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/it_core_news_sm-3.8.0/it_core_news_sm-3.8.0-py3-none-any.whl -pl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/pl_core_news_sm-3.8.0/pl_core_news_sm-3.8.0-py3-none-any.whl -nl-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/nl_core_news_sm-3.8.0/nl_core_news_sm-3.8.0-py3-none-any.whl -pt-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.8.0/pt_core_news_sm-3.8.0-py3-none-any.whl \ No newline at end of file +spacy \ No newline at end of file diff --git a/scripts/restore_punctuation.py b/scripts/restore_punctuation.py index 3f644d51..34039bdc 100644 --- a/scripts/restore_punctuation.py +++ b/scripts/restore_punctuation.py @@ -5,11 +5,13 @@ import spacy from typing import Dict, Callable, List import re +from spacy.cli import download nlp_models: Dict[str, spacy.language.Language] = {} def load_spacy_model(lang_code: str) -> spacy.language.Language: - """Load and return the appropriate spaCy model for the given language code.""" + """Load and return the appropriate spaCy model for the given language code. + Downloads the model if not already installed.""" if lang_code not in nlp_models: model_name = { 'ca': 'ca_core_news_sm', @@ -26,7 +28,12 @@ def load_spacy_model(lang_code: str) -> spacy.language.Language: if model_name is None: raise ValueError(f"Unsupported language code: {lang_code}") - nlp_models[lang_code] = spacy.load(model_name) + try: + nlp_models[lang_code] = spacy.load(model_name) + except OSError: + print(f"Downloading {model_name}...") + download(model_name) + nlp_models[lang_code] = spacy.load(model_name) return nlp_models[lang_code] From 91567584c4e60442ea91f609b884b5142924165c Mon Sep 17 00:00:00 2001 From: Paul-HenriBJT Date: Thu, 31 Oct 2024 12:38:44 +0100 Subject: [PATCH 17/17] added num_proc=args.num_proc --- scripts/casing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/casing.py b/scripts/casing.py index e4416257..b0654016 100644 --- a/scripts/casing.py +++ b/scripts/casing.py @@ -41,6 +41,7 @@ def apply_recasing(examples, text_column, description_column): recased_dataset = dataset.map( apply_recasing, batched=True, + num_proc=args.num_proc, batch_size=args.batch_size, fn_kwargs={"text_column": args.text_column, "description_column": args.description_column}, desc="Applying recasing"