From f07341e99a89e163385e8dff3b704d011684d75e Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Thu, 2 May 2024 11:13:45 +0200 Subject: [PATCH 1/2] chore: add repunctuation script. --- dataspeech/gpu_enrichments/repunct_spacy.py | 140 ++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 dataspeech/gpu_enrichments/repunct_spacy.py diff --git a/dataspeech/gpu_enrichments/repunct_spacy.py b/dataspeech/gpu_enrichments/repunct_spacy.py new file mode 100644 index 00000000..6ad7a6da --- /dev/null +++ b/dataspeech/gpu_enrichments/repunct_spacy.py @@ -0,0 +1,140 @@ +import re + +from datasets import load_dataset +from deepmultilingualpunctuation import PunctuationModel +from multiprocess import set_start_method + +from nltk.tokenize import word_tokenize, sent_tokenize +from nltk.tag import pos_tag + +import nltk +import spacy + +# from rpunct import RestorePuncts + +# rpunct = RestorePuncts() + +model = PunctuationModel() + + +ds = load_dataset("ylacombe/mls-eng-tags", split = "train", num_proc=16) + +def truecasing_by_pos(input_text): + + # break input text to sentences + sent_texts = sent_tokenize(input_text) + + full_text = "" + + for sent_text in sent_texts: + # tokenize the text into words + words = word_tokenize(sent_text) + + # apply POS-tagging on words + tagged_words = pos_tag([word.lower() for word in words]) + + # apply capitalization based on POS tags + capitalized_words = [w.capitalize() if t in ["NNP","NNPS"] else w for (w,t) in tagged_words] + + # capitalize first word in sentence + capitalized_words[0] = capitalized_words[0].capitalize() + + # join capitalized words + text_truecase = " ".join(capitalized_words) + + full_text += text_truecase.strip() + + return full_text.strip() + +def true_case(text): + # Split the text into sentences + sentences = nltk.sent_tokenize(text) + + # Process each sentence + true_cased_sentences = [] + for sentence in sentences: + # Tokenize the sentence + tokens = nltk.word_tokenize(sentence) + + # Perform POS tagging + tagged = nltk.pos_tag(tokens) + + # Capitalize the first word of the sentence and NNP and NNPS tags + for i, (word, tag) in enumerate(tagged): + if i == 0 or tag in ('NNP', 'NNPS'): + tagged[i] = (word.capitalize(), tag) + + # Join tokens back into a string, preserving punctuation + true_cased_sentence = ' '.join(word for word, tag in tagged) + + # Remove spaces between punctuations and the preceding word + true_cased_sentence = re.sub(r'(\w) (\W)', r'\1\2', true_cased_sentence) + + true_cased_sentences.append(true_cased_sentence) + + # Join the processed sentences back into a single string + true_cased_text = ' '.join(true_cased_sentences) + + return true_cased_text + +spacy.require_gpu(gpu_id=2) + +# Load the spaCy model +nlp = spacy.load('en_core_web_trf') + +from spacy.util import compile_infix_regex + +def custom_tokenizer(nlp): + infixes = nlp.Defaults.infixes + ['\w+(?:-\w+)+'] + infix_regex = compile_infix_regex(infixes) + return spacy.tokenizer.Tokenizer(nlp.vocab, infix_finditer=infix_regex.finditer) + +# Use the custom tokenizer +nlp.tokenizer = custom_tokenizer(nlp) + +def true_case_spacy(text): + # Process the text with the spaCy model + doc = nlp(text) + + # Initialize an empty list to hold the processed sentences + true_cased_sentences = [] + + # Iterate through the sentences in the Doc object + for sent in doc.sents: + # Initialize an empty list to hold the processed tokens of the current sentence + processed_tokens = [] + + # Iterate through the tokens in the current sentence + for i, token in enumerate(sent): + # Capitalize the first word of the sentence and proper nouns + if i == 0 or token.pos_ == 'PROPN': + processed_tokens.append(token.text.capitalize()) + else: + processed_tokens.append(token.text) + + # Join the processed tokens back into a string + processed_sentence = ' '.join(processed_tokens) + + # Remove spaces between punctuations and the preceding word + processed_sentence = re.sub(r'(\w) (\W)', r'\1\2', processed_sentence) + + # Add the processed sentence to the list of processed sentences + true_cased_sentences.append(processed_sentence) + + # Join the processed sentences back into a single string + true_cased_text = ' '.join(true_cased_sentences) + + return true_cased_text + + +def repunctuation_apply_simple(batch): + + repunct_sample = model.restore_punctuation(batch["text"]) + batch["repunct_text"] = true_case_spacy(repunct_sample) + + return batch + +if __name__ == "__main__": + set_start_method("spawn") + repunct_ds = ds.map(repunctuation_apply_simple, batch_size=1, num_proc=14) + repunct_ds.push_to_hub("reach-vb/mls-eng-tags-spacy-v2", split = "train") \ No newline at end of file From 6401d80a149ce94994840d3fa415fa1117bfbef3 Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Thu, 2 May 2024 11:17:18 +0200 Subject: [PATCH 2/2] chore: remove redundant code. --- dataspeech/gpu_enrichments/repunct_spacy.py | 75 +++------------------ 1 file changed, 9 insertions(+), 66 deletions(-) diff --git a/dataspeech/gpu_enrichments/repunct_spacy.py b/dataspeech/gpu_enrichments/repunct_spacy.py index 6ad7a6da..66a23b27 100644 --- a/dataspeech/gpu_enrichments/repunct_spacy.py +++ b/dataspeech/gpu_enrichments/repunct_spacy.py @@ -10,74 +10,17 @@ import nltk import spacy -# from rpunct import RestorePuncts - -# rpunct = RestorePuncts() - model = PunctuationModel() +dataset_name = "ylacombe/mls-eng-tags" +output_dataset = "reach-vb/mls-eng-tags-spacy-v2" +process_split = "train" +proc = 16 +device_id = 2 -ds = load_dataset("ylacombe/mls-eng-tags", split = "train", num_proc=16) - -def truecasing_by_pos(input_text): - - # break input text to sentences - sent_texts = sent_tokenize(input_text) - - full_text = "" - - for sent_text in sent_texts: - # tokenize the text into words - words = word_tokenize(sent_text) - - # apply POS-tagging on words - tagged_words = pos_tag([word.lower() for word in words]) - - # apply capitalization based on POS tags - capitalized_words = [w.capitalize() if t in ["NNP","NNPS"] else w for (w,t) in tagged_words] - - # capitalize first word in sentence - capitalized_words[0] = capitalized_words[0].capitalize() - - # join capitalized words - text_truecase = " ".join(capitalized_words) - - full_text += text_truecase.strip() - - return full_text.strip() - -def true_case(text): - # Split the text into sentences - sentences = nltk.sent_tokenize(text) - - # Process each sentence - true_cased_sentences = [] - for sentence in sentences: - # Tokenize the sentence - tokens = nltk.word_tokenize(sentence) - - # Perform POS tagging - tagged = nltk.pos_tag(tokens) - - # Capitalize the first word of the sentence and NNP and NNPS tags - for i, (word, tag) in enumerate(tagged): - if i == 0 or tag in ('NNP', 'NNPS'): - tagged[i] = (word.capitalize(), tag) - - # Join tokens back into a string, preserving punctuation - true_cased_sentence = ' '.join(word for word, tag in tagged) - - # Remove spaces between punctuations and the preceding word - true_cased_sentence = re.sub(r'(\w) (\W)', r'\1\2', true_cased_sentence) - - true_cased_sentences.append(true_cased_sentence) - - # Join the processed sentences back into a single string - true_cased_text = ' '.join(true_cased_sentences) - - return true_cased_text +ds = load_dataset(dataset_name, split = process_split, num_proc=proc) -spacy.require_gpu(gpu_id=2) +spacy.require_gpu(gpu_id=device_id) # Load the spaCy model nlp = spacy.load('en_core_web_trf') @@ -136,5 +79,5 @@ def repunctuation_apply_simple(batch): if __name__ == "__main__": set_start_method("spawn") - repunct_ds = ds.map(repunctuation_apply_simple, batch_size=1, num_proc=14) - repunct_ds.push_to_hub("reach-vb/mls-eng-tags-spacy-v2", split = "train") \ No newline at end of file + repunct_ds = ds.map(repunctuation_apply_simple, batch_size=1, num_proc=proc) + repunct_ds.push_to_hub(output_dataset, split = process_split) \ No newline at end of file