nlp_project.py

# -*- coding: utf-8 -*-
"""NLP project.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1-UOSpssDcC0b0_JrMBCBK9P5KYbeMuwi

#**Medical text classification using BERT**

# Links to the models used and dataset

Medical text dataset : https://www.kaggle.com/datasets/chaitanyakck/medical-text/data
'experts_pubmed':'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3'

 'experts_pubmed':'https://tfhub.dev/google/experts/bert/pubmed/2'

# Context and dataset desciption  

Medical abstracts describe the current conditions of a patient.
Doctors routinely scan dozens or hundreds of abstracts each day as they do their rounds in a hospital
and must quickly pick up on the salient information pointing to the patient’s malady.
You are trying to design assistive technology that can identify, with high precision,
the class of problems described in the abstract.
In the given dataset, abstracts from 5 different conditions have been included:
digestive system diseases, cardiovascular diseases, neoplasms, nervous system diseases, and general pathological conditions.
The training dataset consists of 14438 records and the test dataset consists of 14442 records.
The train data has classes whereas, the test data classes are needed to be predicted.
The data are provided as text in train.dat and test.dat, which should be processed appropriately.

# Related work

Other key words : BERT in clinical NLP", transformer models in healthcare
https://pubmed.ncbi.nlm.nih.gov/?term=BERT+medical+text+classification
https://arxiv.org/search/?query=BERT+medical+text+classification&searchtype=all&source=header
https://consensus.app/results/?q=Performance%20of%20BERT%20in%20medical%20text%20classification%20%3F
"""

# Import libraries

#! pip install pandas matplotlib nltk seaborn requests scikit-learn transformers tensorflow tensorflow_hub
import pandas as pd
import matplotlib.pyplot as plt
from collections import Counter
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
import tensorflow_hub as hub
from sklearn.model_selection import train_test_split

# Data analysis and visualization

train_data_path = 'train_data.txt'
test_data_path = 'test_data.txt'

def split_data_text_labels(train_data_path):
    """This function splits the data into labels and text data"""
    # Initialize empty lists to store labels and text data
    labels = []
    text_data = []

    with open(train_data_path, "r") as f:
        lines = f.readlines()

    # Loop through each line in the file
    for line in lines:
        # Split the line into label and text data
        label, text = line.split('\t', 1)

        # Append the label and text data to their respective lists
        labels.append(int(label))  # Convert label to integer
        text_data.append(text.strip())  # Remove leading/trailing whitespace

    return labels, text_data
def create_df(labels, text_data):
    """This function creates a dataframe with labels and text data"""
    import pandas as pd
    df = pd.DataFrame(list(zip(text_data, labels)), columns =['text', 'label'])
    return df

labels, text_data = split_data_text_labels(train_data_path)
df = create_df(labels, text_data)

# Display the distribution of classes
class_distribution = df['label'].value_counts()
sns.barplot(x=class_distribution.index, y=class_distribution.values)
plt.title('Class Distribution')
plt.ylabel('Number of Occurrences', fontsize=12)
plt.xlabel('Classes', fontsize=12)
plt.show()

# Check for balance in the dataset
print("Class balance:")
print(class_distribution)

# Tokenize the text and remove stopwords
nltk.download('punkt')
nltk.download('stopwords')
stop_words = set(stopwords.words('english'))

# Extend the stop words list if necessary with domain-specific terms
# stop_words.update(['additional', 'stopwords'])

# Tokenize and remove stop words
df['tokens'] = df['text'].apply(lambda x: [word for word in word_tokenize(x.lower()) if word.isalpha()])
df['tokens'] = df['tokens'].apply(lambda x: [word for word in x if word not in stop_words])

# Find the most common words across all classes
all_words = [word for tokens in df['tokens'] for word in tokens]
word_counts = Counter(all_words)

# Get the 20 most common words and their counts
most_common_words = word_counts.most_common(20)
words, counts = zip(*most_common_words)

# Create a bar plot
plt.figure(figsize=(10, 5))
plt.bar(words, counts)
plt.title('Word Frequencies')
plt.xlabel('Words')
plt.ylabel('Frequency')
plt.xticks(rotation=90)
plt.show()

"""
# Model explanation

PubMed Experts
These models were pre-trained on the MEDLINE/PubMed corpus of biomedical and life sciences literature abstracts.
The models are intended to be used on medical or scientific text NLP tasks.
Model	Description
pubmed : The model was trained in a self-supervised manner on the MEDLINE/Pubmed corpus.
pubmed/squad2 : This model was initialized from the base PubMed model and
was fine-tuned on SQuAD 2.0, a dataset for question-answering.

BERT, which stands for Bidirectional Encoder Representations from Transformers,
 is a groundbreaking model in the field of natural language processing (NLP)
   that has set new standards for a variety of language tasks.
   Here's an overview of its architecture and how it can be adapted for a medical text classification task:

**BERT Architecture Overview**:
- **Transformers**: BERT is built on the Transformer architecture, which relies on self-attention mechanisms to process words in relation to all other words in a sentence, contrary to previous models that processed words in order.
- **Bidirectional Context**: Unlike traditional language models that read text from left to right or right to left, BERT reads the entire sequence of words at once. This allows the model to learn the context of a word based on all of its surroundings (left and right of the word).
- **Layers**: BERT models are typically deep neural networks with several layers of transformers. The base BERT model has 12 layers (transformer blocks), while BERT Large has 24.
- **Hidden Units**: Each layer has multiple hidden units. The base model has 768 hidden units, and BERT Large has 1024.
- **Self-Attention Heads**: BERT base has 12 self-attention heads, and BERT Large has 16. These heads allow the model to focus on different parts of the sentence.
- **Input Embeddings**: BERT uses WordPiece embeddings with a 30,000 token vocabulary. It combines these with positional embeddings and segment embeddings to understand the meaning of words in context and differentiate between two sentences when necessary.
- **Pretraining Tasks**: BERT is pretrained on two unsupervised tasks: Masked Language Model (MLM) and Next Sentence Prediction (NSP). MLM randomly masks words in a sentence and predicts them, while NSP predicts if a sentence logically follows another.

**Adapting BERT for Medical Text Classification**:
- **Preprocessing**: Medical texts need to be tokenized using BERT's tokenizer, which can handle the WordPiece tokenization. Special tokens like `[CLS]` (beginning of the sequence) and `[SEP]` (separator) are added.
- **Domain-Specific Pretraining**: Although BERT is pretrained on a large corpus, further pretraining on a medical corpus can help the model understand domain-specific language and concepts.
- **Fine-Tuning**: For classification, the `[CLS]` token's final hidden state is used as the aggregate sequence representation for classification tasks. A simple classification layer is added on top of the BERT model, which is then fine-tuned on the medical text classification dataset.
- **Hyperparameters**: Depending on the size and complexity of the medical dataset, hyperparameters such as batch size, learning rate, and the number of epochs may need to be adjusted during fine-tuning.
- **Handling Imbalance**: Medical datasets can be imbalanced. Techniques like class weighting, oversampling, or focal loss can be used to address this during training.
- **Interpretability**: Given the high stakes of medical decision-making, it's crucial to implement model interpretability techniques to understand the model's predictions.

By adapting BERT with these considerations, we can create a powerful model for classifying medical texts into the desired categories.
It's important to ensure that the model not only performs well in terms of accuracy but also generalizes well to unseen data and maintains a high level of interpretability for end-users,
such as medical professionals.

'''"""

from transformers import AutoTokenizer, AutoModel

# Specify the name of the model
#model_name = 'dmis-lab/biobert-base-cased-v1.1'  # BioBERT
model_name = 'emilyalsentzer/Bio_ClinicalBERT'  # ClinicalBERT

# Load the tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_name)

# Load the model
model = AutoModel.from_pretrained(model_name)

# Print the model's architecture
print(model)

"""# Fine-tune BERT

To fine-tune this model for my task, I
- Made sure my data is in the expected format (a DataFrame with 'text' and 'label' columns).
- Adjusted the `num_labels` argument to match the number of classes in my task.
- Adjusted the training parameters (like the learning rate, number of epochs, and batch size) as needed.
"""

## Split the data into training, validation, and test sets
# Preparing the dataset

train, test = train_test_split(df, test_size=0.2, random_state=42)
train, val = train_test_split(train, test_size=0.2, random_state=42)

print(train.shape)
print(val.shape)
print(test.shape)
print(train.head())

## Prepare the data : load tokenizer and convert data to tensorflow dataset

from transformers import BertTokenizer, TFBertForSequenceClassification
from sklearn.model_selection import train_test_split
import tensorflow as tf

# DataFrame has 'text' and 'label' columns
train_texts = train['text'].tolist()
train_labels = train['label'].tolist()

val_texts = val['text'].tolist()
val_labels = val['label'].tolist()

# Tokenize the texts
tokenizer = BertTokenizer.from_pretrained("medicalai/ClinicalBERT")
train_encodings = tokenizer(train_texts, truncation=True, padding=True, max_length=512)
val_encodings = tokenizer(val_texts, truncation=True, padding=True, max_length=512)

# Convert labels to tensors
train_labels = tf.convert_to_tensor(train_labels)
val_labels = tf.convert_to_tensor(val_labels)

# Convert to TensorFlow Datasets
train_dataset = tf.data.Dataset.from_tensor_slices((dict(train_encodings), train_labels))
val_dataset = tf.data.Dataset.from_tensor_slices((dict(val_encodings), val_labels))
# Convert the datasets to the format expected by the model
def format_dataset(encodings, labels):
    return {'input_ids': encodings['input_ids'], 'attention_mask': encodings['attention_mask']}, labels

train_dataset = train_dataset.map(format_dataset)
val_dataset = val_dataset.map(format_dataset)

# Batch the datasets
train_dataset = train_dataset.shuffle(1000).batch(16)
val_dataset = val_dataset.batch(16)


# Tokenize the test texts
test_texts = test['text'].tolist()
test_labels = test['label'].tolist()
test_encodings = tokenizer(test_texts, truncation=True, padding=True, max_length=512)

# Convert test labels to tensors
test_labels = tf.convert_to_tensor(test_labels)

# Convert to TensorFlow Datasets
test_dataset = tf.data.Dataset.from_tensor_slices((dict(test_encodings), test_labels))

# Format the dataset
test_dataset = test_dataset.map(format_dataset)

# Batch the dataset
test_dataset = test_dataset.batch(16)

## Load the pre-trained model

from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
import tensorflow as tf

# Specify the name of the model
#model_name = 'dmis-lab/biobert-base-cased-v1.1'  # BioBERT
model_name = 'emilyalsentzer/Bio_ClinicalBERT'  # ClinicalBERT

# Load the tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_name)

# Load the model
model = TFAutoModelForSequenceClassification.from_pretrained(model_name, num_labels=5)

# Set up the training parameters
optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
model.compile(optimizer=optimizer, loss=model.compute_loss, metrics=['accuracy'])

# Train the model
model.fit(train_dataset, epochs=3, validation_data=val_dataset)

# Save the model
model.save_pretrained('./medicalai_ClinicalBERT')

model.evaluate(test_dataset)