From 068bf3a6c7462d6f8c5cc812443dddc8ca733817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Stefa=C5=84ski?= Date: Mon, 29 May 2023 13:15:34 +0200 Subject: [PATCH 1/9] added utils to clear project and small return fixes --- data/spectrograms/.gitkeep | 0 requirements.txt | 6 +++++- src/models/.gitkeep | 0 src/models/AudioProcessor.py | 6 +++--- src/models/spectrogram_dataset.py | 2 ++ src/models/train_bird_classifier.py | 3 +++ src/utils/clear_project.py | 19 +++++++++++++++++++ 7 files changed, 32 insertions(+), 4 deletions(-) delete mode 100644 data/spectrograms/.gitkeep delete mode 100644 src/models/.gitkeep create mode 100644 src/utils/clear_project.py diff --git a/data/spectrograms/.gitkeep b/data/spectrograms/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/requirements.txt b/requirements.txt index 29db34c..78de652 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,11 @@ librosa==0.10.0.post2 matplotlib==3.5.3 noisereduce==2.0.1 -numpy==1.23.3 pydub==0.25.1 +pytorch_lightning==2.0.2 requests==2.28.1 +scikit_learn==1.2.2 +scipy==1.10.1 +seaborn==0.12.2 soundfile==0.12.1 +torch==2.0.0 diff --git a/src/models/.gitkeep b/src/models/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/models/AudioProcessor.py b/src/models/AudioProcessor.py index 615ab09..a7ef91f 100644 --- a/src/models/AudioProcessor.py +++ b/src/models/AudioProcessor.py @@ -55,7 +55,7 @@ def load_and_trim_audio(self, file_path): audio, sr = librosa.load(file_path, sr=self.sample_rate, duration=self.fixed_length_seconds) if len(audio) == 0: - return None + return None, None audio = self.trim_audio(audio, sr) @@ -65,11 +65,11 @@ def preprocess_audio(self, file_path): audio, sr = librosa.load(file_path, sr=self.sample_rate, duration=self.fixed_length_seconds) if len(audio) == 0: - return None + return None, None audio = self.trim_audio(audio, sr) if audio is None: - return None + return None, None audio = nr.reduce_noise(y=audio, sr=sr) if self.normalize_volume: audio = librosa.util.normalize(audio) diff --git a/src/models/spectrogram_dataset.py b/src/models/spectrogram_dataset.py index 44befe7..457b45b 100644 --- a/src/models/spectrogram_dataset.py +++ b/src/models/spectrogram_dataset.py @@ -19,6 +19,8 @@ def __init__(self, root_dir, split="train"): if class_dir.name not in self.label_map: self.label_map[class_dir.name] = len(self.label_map) for file in class_dir.glob("*.mp3"): + if file is None: + continue trimed_audio, sr = self.audio_preprocessor.load_and_trim_audio(file) if trimed_audio is None: continue diff --git a/src/models/train_bird_classifier.py b/src/models/train_bird_classifier.py index edb02cc..6c953fb 100644 --- a/src/models/train_bird_classifier.py +++ b/src/models/train_bird_classifier.py @@ -3,6 +3,8 @@ from bird_spectrogram_classifier import BirdClassifier from spectrogram_dataset import BirdSpectrogramDataModule +import wandb + # Configuration parameters root_dirs = { "split1": "data/split_raw", @@ -29,3 +31,4 @@ # Save the trained model trainer.save_checkpoint(f"{split_name}_bird_classifier.ckpt") + wandb_logger.experiment.finish() diff --git a/src/utils/clear_project.py b/src/utils/clear_project.py new file mode 100644 index 0000000..41ef36b --- /dev/null +++ b/src/utils/clear_project.py @@ -0,0 +1,19 @@ +import shutil + +numpy_path = "data/numpy" +processed_path = "data/processed" +spectrograms_path = "data/spectrograms" +split_aug_path = "data/split_aug" +split_raw_path = "data/split_raw" +split_wrong_path = "data/split_wrong" +bird_classification_path = "bird-classification" + + +# Delete the folder and its contents +shutil.rmtree(numpy_path) +shutil.rmtree(processed_path) +shutil.rmtree(spectrograms_path) +shutil.rmtree(split_aug_path) +shutil.rmtree(split_raw_path) +shutil.rmtree(split_wrong_path) +shutil.rmtree(bird_classification_path) From 0b18e51a77a032ea05ca339c418b0ee6df1ccddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Stefa=C5=84ski?= Date: Mon, 29 May 2023 14:35:06 +0200 Subject: [PATCH 2/9] horus: added a pretrained resnet --- src/models/BirdClassifierResNet.py | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/models/BirdClassifierResNet.py diff --git a/src/models/BirdClassifierResNet.py b/src/models/BirdClassifierResNet.py new file mode 100644 index 0000000..b63304d --- /dev/null +++ b/src/models/BirdClassifierResNet.py @@ -0,0 +1,40 @@ +import torch +from torch import nn +from torch.nn import functional as F +import pytorch_lightning as pl +from torchvision import models + +class BirdClassifierResNet(pl.LightningModule): + def __init__(self, num_classes): + super(BirdClassifierResNet, self).__init__() + + # Load pre-trained ResNet50 + self.feature_extractor = models.resnet50(pretrained=True) + + # Replace the last fully connected layer + # Parameters of newly constructed modules have requires_grad=True by default + num_ftrs = self.feature_extractor.fc.in_features + self.feature_extractor.fc = nn.Linear(num_ftrs, num_classes) + + def forward(self, x): + return self.feature_extractor(x) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.cross_entropy(logits, y) + self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.cross_entropy(logits, y) + preds = torch.argmax(logits, dim=1) + acc = (preds == y).float().mean() + metrics = {"val_loss": loss, "val_acc": acc} + self.log_dict(metrics, on_step=False, on_epoch=True, prog_bar=True, logger=True) + return metrics + + def configure_optimizers(self): + return torch.optim.Adam(self.parameters(), lr=0.001) From 582bbbae908aa2470d6588d5051280e24ec29185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Stefa=C5=84ski?= Date: Mon, 29 May 2023 15:19:25 +0200 Subject: [PATCH 3/9] horus: added early stopping, fixed checkpoint cb --- src/models/BirdClassifierResNet.py | 6 ++-- src/models/spectrogram_dataset.py | 9 +++--- src/models/train_bird_classifier.py | 49 +++++++++++++++++++++-------- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/src/models/BirdClassifierResNet.py b/src/models/BirdClassifierResNet.py index b63304d..6fc6d0f 100644 --- a/src/models/BirdClassifierResNet.py +++ b/src/models/BirdClassifierResNet.py @@ -2,15 +2,15 @@ from torch import nn from torch.nn import functional as F import pytorch_lightning as pl -from torchvision import models +from torchvision.models.resnet import ResNet, ResNet50_Weights +from torchvision.models import resnet50 class BirdClassifierResNet(pl.LightningModule): def __init__(self, num_classes): super(BirdClassifierResNet, self).__init__() # Load pre-trained ResNet50 - self.feature_extractor = models.resnet50(pretrained=True) - + self.feature_extractor = resnet50(weights=ResNet50_Weights.DEFAULT) # Replace the last fully connected layer # Parameters of newly constructed modules have requires_grad=True by default num_ftrs = self.feature_extractor.fc.in_features diff --git a/src/models/spectrogram_dataset.py b/src/models/spectrogram_dataset.py index 457b45b..03e7ce7 100644 --- a/src/models/spectrogram_dataset.py +++ b/src/models/spectrogram_dataset.py @@ -37,10 +37,11 @@ def __getitem__(self, idx): return mel_spec, label class BirdSpectrogramDataModule(pl.LightningDataModule): - def __init__(self, root_dir, batch_size=16): + def __init__(self, root_dir, batch_size=16, num_workers=4): super().__init__() self.root_dir = root_dir self.batch_size = batch_size + self.num_workers = num_workers def setup(self, stage=None): if stage == "fit" or stage is None: @@ -56,10 +57,10 @@ def _create_datasets(self, *splits): return datasets def train_dataloader(self): - return DataLoader(self.train_dataset, batch_size=self.batch_size, shuffle=True) + return DataLoader(self.train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers) def val_dataloader(self): - return DataLoader(self.val_dataset, batch_size=self.batch_size, shuffle=False) + return DataLoader(self.val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers) def test_dataloader(self): - return DataLoader(self.test_dataset, batch_size=self.batch_size, shuffle=False) + return DataLoader(self.test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers) diff --git a/src/models/train_bird_classifier.py b/src/models/train_bird_classifier.py index 6c953fb..af1d657 100644 --- a/src/models/train_bird_classifier.py +++ b/src/models/train_bird_classifier.py @@ -1,6 +1,8 @@ import pytorch_lightning as pl +from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping from pytorch_lightning.loggers import WandbLogger from bird_spectrogram_classifier import BirdClassifier +from BirdClassifierResNet import BirdClassifierResNet from spectrogram_dataset import BirdSpectrogramDataModule import wandb @@ -15,20 +17,41 @@ batch_size = 16 max_epochs = 10 -for split_name, root_dir in root_dirs.items(): - # Initialize the DataModule - data_module = BirdSpectrogramDataModule(root_dir, batch_size=batch_size) +def main(): + # Initialize the early stopping callback + early_stopping = EarlyStopping(monitor='val_loss', patience=3, verbose=True, mode='min') - # Initialize the model - model = BirdClassifier(num_classes=num_classes) + for split_name, root_dir in root_dirs.items(): + # Initialize the DataModule + data_module = BirdSpectrogramDataModule(root_dir, batch_size=batch_size) - # Set up the Weights & Biases logger - wandb_logger = WandbLogger(project="bird-classification", name=f"{split_name}_run") + # Initialize the model + model = BirdClassifier(num_classes=num_classes) - # Train the model - trainer = pl.Trainer(max_epochs=max_epochs, logger=wandb_logger, log_every_n_steps=1) - trainer.fit(model, data_module) + # Set up the Weights & Biases logger + wandb_logger = WandbLogger(project="bird-classification", name=f"{split_name}_run") - # Save the trained model - trainer.save_checkpoint(f"{split_name}_bird_classifier.ckpt") - wandb_logger.experiment.finish() + # Define checkpoint callback + checkpoint_callback = ModelCheckpoint( + filename=f"{split_name}_bird_classifier", + monitor='val_loss', # The metric to monitor + save_top_k=1, # Save only the top 1 models based on the metric monitored + mode='min', # In 'min' mode, training will stop when the quantity monitored has stopped decreasing + verbose=True # Report when a new best model is saved + ) + + # Train the model + trainer = pl.Trainer( + max_epochs=max_epochs, + logger=wandb_logger, + callbacks=[checkpoint_callback, early_stopping], # Add callbacks here + log_every_n_steps=1, + ) + trainer.fit(model, data_module) + + # Save the trained model (The best model is saved by the checkpoint_callback) + # trainer.save_checkpoint(f"{split_name}_bird_classifier.ckpt") + wandb_logger.experiment.finish() + +if __name__ == "__main__": + main() From be73ff6c51140eb87b3ceeca104891b188394609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Stefa=C5=84ski?= Date: Mon, 29 May 2023 15:29:05 +0200 Subject: [PATCH 4/9] Horus: added support for multiple batch sizes --- src/models/train_bird_classifier.py | 65 +++++++++++++++-------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/src/models/train_bird_classifier.py b/src/models/train_bird_classifier.py index af1d657..554a78a 100644 --- a/src/models/train_bird_classifier.py +++ b/src/models/train_bird_classifier.py @@ -14,44 +14,45 @@ "split3": "data/split_wrong", } num_classes = 7 # Replace with the number of bird species -batch_size = 16 +batch_sizes = [16, 32, 64] # list of batch sizes you want to run with max_epochs = 10 def main(): # Initialize the early stopping callback early_stopping = EarlyStopping(monitor='val_loss', patience=3, verbose=True, mode='min') - for split_name, root_dir in root_dirs.items(): - # Initialize the DataModule - data_module = BirdSpectrogramDataModule(root_dir, batch_size=batch_size) - - # Initialize the model - model = BirdClassifier(num_classes=num_classes) - - # Set up the Weights & Biases logger - wandb_logger = WandbLogger(project="bird-classification", name=f"{split_name}_run") - - # Define checkpoint callback - checkpoint_callback = ModelCheckpoint( - filename=f"{split_name}_bird_classifier", - monitor='val_loss', # The metric to monitor - save_top_k=1, # Save only the top 1 models based on the metric monitored - mode='min', # In 'min' mode, training will stop when the quantity monitored has stopped decreasing - verbose=True # Report when a new best model is saved - ) - - # Train the model - trainer = pl.Trainer( - max_epochs=max_epochs, - logger=wandb_logger, - callbacks=[checkpoint_callback, early_stopping], # Add callbacks here - log_every_n_steps=1, - ) - trainer.fit(model, data_module) - - # Save the trained model (The best model is saved by the checkpoint_callback) - # trainer.save_checkpoint(f"{split_name}_bird_classifier.ckpt") - wandb_logger.experiment.finish() + for batch_size in batch_sizes: + for split_name, root_dir in root_dirs.items(): + # Initialize the DataModule + data_module = BirdSpectrogramDataModule(root_dir, batch_size=batch_size) + + # Initialize the model + model = BirdClassifier(num_classes=num_classes) + + # Set up the Weights & Biases logger + wandb_logger = WandbLogger(project="bird-classification", name=f"{split_name}_batch_{batch_size}_run") + + # Define checkpoint callback + checkpoint_callback = ModelCheckpoint( + filename=f"{split_name}_bird_classifier_batch_{batch_size}", + monitor='val_loss', # The metric to monitor + save_top_k=1, # Save only the top 1 models based on the metric monitored + mode='min', # In 'min' mode, training will stop when the quantity monitored has stopped decreasing + verbose=True # Report when a new best model is saved + ) + + # Train the model + trainer = pl.Trainer( + max_epochs=max_epochs, + logger=wandb_logger, + callbacks=[checkpoint_callback, early_stopping], # Add callbacks here + log_every_n_steps=1, + ) + trainer.fit(model, data_module) + + # Save the trained model (The best model is saved by the checkpoint_callback) + # trainer.save_checkpoint(f"{split_name}_bird_classifier_batch_{batch_size}.ckpt") + wandb_logger.experiment.finish() if __name__ == "__main__": main() From b2bb4b15191a64552698561032629b34919a8655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Stefa=C5=84ski?= Date: Mon, 29 May 2023 15:32:59 +0200 Subject: [PATCH 5/9] Horus: added support for multiple learning rates --- src/models/BirdClassifierResNet.py | 6 ++- src/models/train_bird_classifier.py | 69 ++++++++++++++++------------- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/models/BirdClassifierResNet.py b/src/models/BirdClassifierResNet.py index 6fc6d0f..1acaf70 100644 --- a/src/models/BirdClassifierResNet.py +++ b/src/models/BirdClassifierResNet.py @@ -6,7 +6,7 @@ from torchvision.models import resnet50 class BirdClassifierResNet(pl.LightningModule): - def __init__(self, num_classes): + def __init__(self, num_classes, learning_rate=0.001): super(BirdClassifierResNet, self).__init__() # Load pre-trained ResNet50 @@ -15,6 +15,8 @@ def __init__(self, num_classes): # Parameters of newly constructed modules have requires_grad=True by default num_ftrs = self.feature_extractor.fc.in_features self.feature_extractor.fc = nn.Linear(num_ftrs, num_classes) + + self.learning_rate = learning_rate def forward(self, x): return self.feature_extractor(x) @@ -37,4 +39,4 @@ def validation_step(self, batch, batch_idx): return metrics def configure_optimizers(self): - return torch.optim.Adam(self.parameters(), lr=0.001) + return torch.optim.Adam(self.parameters(), lr=self.learning_rate) diff --git a/src/models/train_bird_classifier.py b/src/models/train_bird_classifier.py index 554a78a..12b6e7f 100644 --- a/src/models/train_bird_classifier.py +++ b/src/models/train_bird_classifier.py @@ -16,43 +16,48 @@ num_classes = 7 # Replace with the number of bird species batch_sizes = [16, 32, 64] # list of batch sizes you want to run with max_epochs = 10 +learning_rates = [0.001, 0.01, 0.1] def main(): # Initialize the early stopping callback early_stopping = EarlyStopping(monitor='val_loss', patience=3, verbose=True, mode='min') - for batch_size in batch_sizes: - for split_name, root_dir in root_dirs.items(): - # Initialize the DataModule - data_module = BirdSpectrogramDataModule(root_dir, batch_size=batch_size) - - # Initialize the model - model = BirdClassifier(num_classes=num_classes) - - # Set up the Weights & Biases logger - wandb_logger = WandbLogger(project="bird-classification", name=f"{split_name}_batch_{batch_size}_run") - - # Define checkpoint callback - checkpoint_callback = ModelCheckpoint( - filename=f"{split_name}_bird_classifier_batch_{batch_size}", - monitor='val_loss', # The metric to monitor - save_top_k=1, # Save only the top 1 models based on the metric monitored - mode='min', # In 'min' mode, training will stop when the quantity monitored has stopped decreasing - verbose=True # Report when a new best model is saved - ) - - # Train the model - trainer = pl.Trainer( - max_epochs=max_epochs, - logger=wandb_logger, - callbacks=[checkpoint_callback, early_stopping], # Add callbacks here - log_every_n_steps=1, - ) - trainer.fit(model, data_module) - - # Save the trained model (The best model is saved by the checkpoint_callback) - # trainer.save_checkpoint(f"{split_name}_bird_classifier_batch_{batch_size}.ckpt") - wandb_logger.experiment.finish() + for learning_rate in learning_rates: + for batch_size in batch_sizes: + for split_name, root_dir in root_dirs.items(): + # Initialize the DataModule + data_module = BirdSpectrogramDataModule(root_dir, batch_size=batch_size) + + # Initialize the model + model = BirdClassifier(num_classes=num_classes, learning_rate=learning_rate) + + # Set up the Weights & Biases logger + wandb_logger = WandbLogger( + project="bird-classification", + name=f"{split_name}_batch_{batch_size}_lr_{learning_rate}_run" + ) + + # Define checkpoint callback + checkpoint_callback = ModelCheckpoint( + filename=f"{split_name}_bird_classifier_batch_{batch_size}_lr_{learning_rate}", + monitor='val_loss', # The metric to monitor + save_top_k=1, # Save only the top 1 models based on the metric monitored + mode='min', # In 'min' mode, training will stop when the quantity monitored has stopped decreasing + verbose=True # Report when a new best model is saved + ) + + # Train the model + trainer = pl.Trainer( + max_epochs=max_epochs, + logger=wandb_logger, + callbacks=[checkpoint_callback, early_stopping], # Add callbacks here + log_every_n_steps=1, + ) + trainer.fit(model, data_module) + + # Save the trained model (The best model is saved by the checkpoint_callback) + # trainer.save_checkpoint(f"{split_name}_bird_classifier_batch_{batch_size}.ckpt") + wandb_logger.experiment.finish() if __name__ == "__main__": main() From 726573f1203e12de29e8d962be191dd9f5ef0a26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Stefa=C5=84ski?= Date: Mon, 29 May 2023 15:33:53 +0200 Subject: [PATCH 6/9] Horus: reformatted the project to meet pep8 standards --- src/models/AudioProcessor.py | 27 ++++++++++------ src/models/BirdClassifierResNet.py | 7 +++-- src/models/bird_spectrogram_classifier.py | 9 ++++-- src/models/spectrogram_dataset.py | 32 ++++++++++++++++--- src/models/train_bird_classifier.py | 27 ++++++++++------ src/utils/correlation_calculation.py | 38 ++++++++++++++++------- src/utils/split_dataset.py | 23 +++++++------- 7 files changed, 113 insertions(+), 50 deletions(-) diff --git a/src/models/AudioProcessor.py b/src/models/AudioProcessor.py index a7ef91f..5832b8d 100644 --- a/src/models/AudioProcessor.py +++ b/src/models/AudioProcessor.py @@ -39,7 +39,7 @@ def shift_pitch(self, audio, sr, pitch_factor=3): def change_speed(self, audio, sr, speed_factor=1.2): return librosa.effects.time_stretch(audio, rate=speed_factor) - + def trim_audio(self, audio, sr): audio_length = len(audio) fixed_length_samples = self.fixed_length_seconds * sr @@ -50,9 +50,11 @@ def trim_audio(self, audio, sr): audio = audio[:fixed_length_samples] return audio - + def load_and_trim_audio(self, file_path): - audio, sr = librosa.load(file_path, sr=self.sample_rate, duration=self.fixed_length_seconds) + audio, sr = librosa.load( + file_path, sr=self.sample_rate, duration=self.fixed_length_seconds + ) if len(audio) == 0: return None, None @@ -62,11 +64,13 @@ def load_and_trim_audio(self, file_path): return audio, sr def preprocess_audio(self, file_path): - audio, sr = librosa.load(file_path, sr=self.sample_rate, duration=self.fixed_length_seconds) + audio, sr = librosa.load( + file_path, sr=self.sample_rate, duration=self.fixed_length_seconds + ) if len(audio) == 0: return None, None - + audio = self.trim_audio(audio, sr) if audio is None: return None, None @@ -89,7 +93,9 @@ def save_preprocessed_audio(self, audio, sr, file_path): def process_and_save_audio(self, file_path): preprocessed_audio, sr = self.preprocess_audio(file_path) - target_file_path = self.save_preprocessed_audio(preprocessed_audio, sr, file_path) + target_file_path = self.save_preprocessed_audio( + preprocessed_audio, sr, file_path + ) return target_file_path def create_spectrogram(self, file_path, n_fft=2048, hop_length=512): @@ -109,7 +115,9 @@ def create_mel_spectrogram_from_audio(self, audio, sr, n_mels=128, hop_length=51 def create_mel_spectrogram(self, file_path, n_mels=128, hop_length=512): y, sr = librosa.load(file_path, sr=self.sample_rate) - return self.create_mel_spectrogram_from_audio(y, sr, n_mels=n_mels, hop_length=hop_length) + return self.create_mel_spectrogram_from_audio( + y, sr, n_mels=n_mels, hop_length=hop_length + ) def save_mel_spectrogram(self, mel_spectrogram_db, target_path, sr, hop_length): plt.figure(figsize=(10, 5)) @@ -132,7 +140,8 @@ def save_mel_spectrogram(self, mel_spectrogram_db, target_path, sr, hop_length): mel_spec_path = ( self.numpy_dir / target_path.parent.name - / target_path.with_suffix(".npy").name ) + / target_path.with_suffix(".npy").name + ) plot_path.parent.mkdir(parents=True, exist_ok=True) plt.savefig(plot_path) plt.close() @@ -217,7 +226,7 @@ def process_folder(self, folder_path: Path): except Exception as e: print(f"Error processing file {file_path}: {e}") - + if __name__ == "__main__": audio_preprocessor = AudioPreprocessor() diff --git a/src/models/BirdClassifierResNet.py b/src/models/BirdClassifierResNet.py index 1acaf70..8c352df 100644 --- a/src/models/BirdClassifierResNet.py +++ b/src/models/BirdClassifierResNet.py @@ -5,6 +5,7 @@ from torchvision.models.resnet import ResNet, ResNet50_Weights from torchvision.models import resnet50 + class BirdClassifierResNet(pl.LightningModule): def __init__(self, num_classes, learning_rate=0.001): super(BirdClassifierResNet, self).__init__() @@ -15,7 +16,7 @@ def __init__(self, num_classes, learning_rate=0.001): # Parameters of newly constructed modules have requires_grad=True by default num_ftrs = self.feature_extractor.fc.in_features self.feature_extractor.fc = nn.Linear(num_ftrs, num_classes) - + self.learning_rate = learning_rate def forward(self, x): @@ -25,7 +26,9 @@ def training_step(self, batch, batch_idx): x, y = batch logits = self(x) loss = F.cross_entropy(logits, y) - self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + self.log( + "train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True + ) return loss def validation_step(self, batch, batch_idx): diff --git a/src/models/bird_spectrogram_classifier.py b/src/models/bird_spectrogram_classifier.py index e5b0242..cff1b2d 100644 --- a/src/models/bird_spectrogram_classifier.py +++ b/src/models/bird_spectrogram_classifier.py @@ -3,13 +3,16 @@ from torch.nn import functional as F import pytorch_lightning as pl + class BirdClassifier(pl.LightningModule): def __init__(self, num_classes): super(BirdClassifier, self).__init__() self.conv1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1) self.conv2 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1) self.pool = nn.MaxPool2d(kernel_size=2, stride=2, padding=0) - self.fc1 = nn.Linear(239616, 512) # wartości działające ale z dupy do poprawy TODO + self.fc1 = nn.Linear( + 239616, 512 + ) # wartości działające ale z dupy do poprawy TODO self.fc2 = nn.Linear(512, num_classes) def forward(self, x): @@ -26,7 +29,9 @@ def training_step(self, batch, batch_idx): x, y = batch logits = self(x) loss = F.cross_entropy(logits, y) - self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + self.log( + "train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True + ) return loss def validation_step(self, batch, batch_idx): diff --git a/src/models/spectrogram_dataset.py b/src/models/spectrogram_dataset.py index 03e7ce7..52ce690 100644 --- a/src/models/spectrogram_dataset.py +++ b/src/models/spectrogram_dataset.py @@ -7,6 +7,7 @@ import numpy as np from AudioProcessor import AudioPreprocessor + class BirdSpectrogramDataset(Dataset): def __init__(self, root_dir, split="train"): self.root_dir = Path(root_dir) @@ -24,7 +25,9 @@ def __init__(self, root_dir, split="train"): trimed_audio, sr = self.audio_preprocessor.load_and_trim_audio(file) if trimed_audio is None: continue - mel_spec = self.audio_preprocessor.create_mel_spectrogram_from_audio(trimed_audio, sr) + mel_spec = self.audio_preprocessor.create_mel_spectrogram_from_audio( + trimed_audio, sr + ) label = self.label_map[class_dir.name] self.data.append((mel_spec, label)) @@ -34,8 +37,10 @@ def __len__(self): def __getitem__(self, idx): mel_spec, label = self.data[idx] mel_spec = torch.from_numpy(mel_spec).unsqueeze(0) # Add channel dimension + mel_spec = mel_spec.expand(3, -1, -1) # Expand single channel to three channels return mel_spec, label + class BirdSpectrogramDataModule(pl.LightningDataModule): def __init__(self, root_dir, batch_size=16, num_workers=4): super().__init__() @@ -45,7 +50,9 @@ def __init__(self, root_dir, batch_size=16, num_workers=4): def setup(self, stage=None): if stage == "fit" or stage is None: - self.train_dataset, self.val_dataset = self._create_datasets("train", "validation") + self.train_dataset, self.val_dataset = self._create_datasets( + "train", "validation" + ) if stage == "test" or stage is None: self.test_dataset, _ = self._create_datasets("test") @@ -57,10 +64,25 @@ def _create_datasets(self, *splits): return datasets def train_dataloader(self): - return DataLoader(self.train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers) + return DataLoader( + self.train_dataset, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + ) def val_dataloader(self): - return DataLoader(self.val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers) + return DataLoader( + self.val_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + ) def test_dataloader(self): - return DataLoader(self.test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers) + return DataLoader( + self.test_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + ) diff --git a/src/models/train_bird_classifier.py b/src/models/train_bird_classifier.py index 12b6e7f..cd39b9e 100644 --- a/src/models/train_bird_classifier.py +++ b/src/models/train_bird_classifier.py @@ -13,14 +13,17 @@ "split2": "data/split_aug", "split3": "data/split_wrong", } -num_classes = 7 # Replace with the number of bird species +num_classes = 7 # Replace with the number of bird species batch_sizes = [16, 32, 64] # list of batch sizes you want to run with max_epochs = 10 -learning_rates = [0.001, 0.01, 0.1] +learning_rates = [0.001, 0.01, 0.1] + def main(): # Initialize the early stopping callback - early_stopping = EarlyStopping(monitor='val_loss', patience=3, verbose=True, mode='min') + early_stopping = EarlyStopping( + monitor="val_loss", patience=3, verbose=True, mode="min" + ) for learning_rate in learning_rates: for batch_size in batch_sizes: @@ -29,28 +32,33 @@ def main(): data_module = BirdSpectrogramDataModule(root_dir, batch_size=batch_size) # Initialize the model - model = BirdClassifier(num_classes=num_classes, learning_rate=learning_rate) + model = BirdClassifierResNet( + num_classes=num_classes, learning_rate=learning_rate + ) # Set up the Weights & Biases logger wandb_logger = WandbLogger( project="bird-classification", - name=f"{split_name}_batch_{batch_size}_lr_{learning_rate}_run" + name=f"{split_name}_batch_{batch_size}_lr_{learning_rate}_run", ) # Define checkpoint callback checkpoint_callback = ModelCheckpoint( filename=f"{split_name}_bird_classifier_batch_{batch_size}_lr_{learning_rate}", - monitor='val_loss', # The metric to monitor + monitor="val_loss", # The metric to monitor save_top_k=1, # Save only the top 1 models based on the metric monitored - mode='min', # In 'min' mode, training will stop when the quantity monitored has stopped decreasing - verbose=True # Report when a new best model is saved + mode="min", # In 'min' mode, training will stop when the quantity monitored has stopped decreasing + verbose=True, # Report when a new best model is saved ) # Train the model trainer = pl.Trainer( max_epochs=max_epochs, logger=wandb_logger, - callbacks=[checkpoint_callback, early_stopping], # Add callbacks here + callbacks=[ + checkpoint_callback, + early_stopping, + ], # Add callbacks here log_every_n_steps=1, ) trainer.fit(model, data_module) @@ -59,5 +67,6 @@ def main(): # trainer.save_checkpoint(f"{split_name}_bird_classifier_batch_{batch_size}.ckpt") wandb_logger.experiment.finish() + if __name__ == "__main__": main() diff --git a/src/utils/correlation_calculation.py b/src/utils/correlation_calculation.py index 2e80730..e1e1fb7 100644 --- a/src/utils/correlation_calculation.py +++ b/src/utils/correlation_calculation.py @@ -7,29 +7,35 @@ import matplotlib.pyplot as plt from scipy.interpolate import interp2d -base_path = Path('data/numpy') +base_path = Path("data/numpy") # Create a list of folder names -folders = [os.path.join(base_path, f) for f in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, f))] +folders = [ + os.path.join(base_path, f) + for f in os.listdir(base_path) + if os.path.isdir(os.path.join(base_path, f)) +] species_names = [os.path.basename(f) for f in folders] # Initialize lists to store the data spectrogram_data = [] species_labels = [] + # Function to resample spectrogram def resample_spectrogram(spectrogram, new_shape=(128, 128)): x = np.linspace(0, spectrogram.shape[1], num=spectrogram.shape[1]) y = np.linspace(0, spectrogram.shape[0], num=spectrogram.shape[0]) - interpolator = interp2d(x, y, spectrogram, kind='linear') + interpolator = interp2d(x, y, spectrogram, kind="linear") x_new = np.linspace(0, spectrogram.shape[1], num=new_shape[1]) y_new = np.linspace(0, spectrogram.shape[0], num=new_shape[0]) resampled_spectrogram = interpolator(x_new, y_new) return resampled_spectrogram + # Load the data from .npy files for idx, folder in enumerate(folders): - file_list = glob.glob(os.path.join(folder, '*.npy')) - + file_list = glob.glob(os.path.join(folder, "*.npy")) + for file in file_list: spectrogram = np.load(file) resampled_spectrogram = resample_spectrogram(spectrogram) @@ -40,7 +46,9 @@ def resample_spectrogram(spectrogram, new_shape=(128, 128)): avg_spectrograms = [] for idx in range(len(folders)): - species_spectrograms = [spec for spec, label in zip(spectrogram_data, species_labels) if label == idx] + species_spectrograms = [ + spec for spec, label in zip(spectrogram_data, species_labels) if label == idx + ] avg_spectrogram = np.mean(species_spectrograms, axis=0) avg_spectrograms.append(avg_spectrogram) @@ -50,14 +58,22 @@ def resample_spectrogram(spectrogram, new_shape=(128, 128)): for i in range(len(folders)): for j in range(len(folders)): # Calculate the Pearson correlation coefficient between the average spectrograms - corr, _ = stats.pearsonr(avg_spectrograms[i].flatten(), avg_spectrograms[j].flatten()) + corr, _ = stats.pearsonr( + avg_spectrograms[i].flatten(), avg_spectrograms[j].flatten() + ) correlation_matrix[i, j] = corr # Visualize the correlation matrix using a heatmap plt.figure(figsize=(10, 8)) sns.set(font_scale=1.2) -heatmap = sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', xticklabels=species_names, yticklabels=species_names) -plt.title('Correlation Matrix of Average Spectrograms') -plt.xlabel('Species') -plt.ylabel('Species') +heatmap = sns.heatmap( + correlation_matrix, + annot=True, + cmap="coolwarm", + xticklabels=species_names, + yticklabels=species_names, +) +plt.title("Correlation Matrix of Average Spectrograms") +plt.xlabel("Species") +plt.ylabel("Species") plt.show() diff --git a/src/utils/split_dataset.py b/src/utils/split_dataset.py index ae49e76..8372166 100644 --- a/src/utils/split_dataset.py +++ b/src/utils/split_dataset.py @@ -2,6 +2,7 @@ from sklearn.model_selection import train_test_split import shutil + def save_split_files(sets, output_directory): for set_name, files in sets.items(): set_path = os.path.join(output_directory, set_name) @@ -15,29 +16,27 @@ def save_split_files(sets, output_directory): os.makedirs(class_path, exist_ok=True) shutil.copy(file, os.path.join(class_path, file_name)) - -def split_dataset(source_directory, output_directory, wrong_split=False, train_size=0.7): + +def split_dataset( + source_directory, output_directory, wrong_split=False, train_size=0.7 +): all_files = [] - for root,_, files in os.walk(source_directory): + for root, _, files in os.walk(source_directory): for file in files: if file.endswith(".mp3"): all_files.append(os.path.join(root, file)) - train, remaining= train_test_split(all_files, train_size=train_size) + train, remaining = train_test_split(all_files, train_size=train_size) valid, test = train_test_split(remaining, test_size=0.5) if wrong_split: train = train + valid - sets = { - "train": train, - "validation": valid, - "test": test - } - + sets = {"train": train, "validation": valid, "test": test} + save_split_files(sets, output_directory) - + split_dataset("data/processed", "data/split_aug") split_dataset("data/raw", "data/split_raw") -split_dataset("data/processed", "data/split_wrong", wrong_split=True) \ No newline at end of file +split_dataset("data/processed", "data/split_wrong", wrong_split=True) From 9aa93c7b2ae060fbc0f6004bfd316037e56e0b1c Mon Sep 17 00:00:00 2001 From: Jereml Date: Fri, 2 Jun 2023 22:24:30 +0200 Subject: [PATCH 7/9] Remove Early stopping --- src/models/train_bird_classifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/train_bird_classifier.py b/src/models/train_bird_classifier.py index cd39b9e..a5e750e 100644 --- a/src/models/train_bird_classifier.py +++ b/src/models/train_bird_classifier.py @@ -57,7 +57,7 @@ def main(): logger=wandb_logger, callbacks=[ checkpoint_callback, - early_stopping, + #early_stopping, ], # Add callbacks here log_every_n_steps=1, ) From 0552e6364bd0397e04de31aaf8a2de0552e7b246 Mon Sep 17 00:00:00 2001 From: Jereml Date: Fri, 2 Jun 2023 22:24:45 +0200 Subject: [PATCH 8/9] Add more metric --- src/models/BirdClassifierResNet.py | 44 ++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/models/BirdClassifierResNet.py b/src/models/BirdClassifierResNet.py index 8c352df..1b28f42 100644 --- a/src/models/BirdClassifierResNet.py +++ b/src/models/BirdClassifierResNet.py @@ -4,7 +4,7 @@ import pytorch_lightning as pl from torchvision.models.resnet import ResNet, ResNet50_Weights from torchvision.models import resnet50 - +from torchmetrics import Precision, Accuracy, F1Score, Recall, ConfusionMatrix, AUROC class BirdClassifierResNet(pl.LightningModule): def __init__(self, num_classes, learning_rate=0.001): @@ -19,6 +19,14 @@ def __init__(self, num_classes, learning_rate=0.001): self.learning_rate = learning_rate + # Define metrics + self.train_acc = Accuracy() + self.val_acc = Accuracy() + self.precision_metric = Precision(num_classes=num_classes) + self.recall = Recall(num_classes=num_classes) + self.f1 = F1Score(num_classes=num_classes) + self.auroc = AUROC(num_classes=num_classes) + def forward(self, x): return self.feature_extractor(x) @@ -26,9 +34,16 @@ def training_step(self, batch, batch_idx): x, y = batch logits = self(x) loss = F.cross_entropy(logits, y) - self.log( - "train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True - ) + preds = torch.argmax(logits, dim=1) + + # Update metrics + self.train_acc(preds, y) + self.precision_metric(preds, y) + self.recall(preds, y) + self.f1(preds, y) + + self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + self.log('train_acc', self.train_acc, on_step=True, on_epoch=True, prog_bar=True, logger=True) return loss def validation_step(self, batch, batch_idx): @@ -36,10 +51,23 @@ def validation_step(self, batch, batch_idx): logits = self(x) loss = F.cross_entropy(logits, y) preds = torch.argmax(logits, dim=1) - acc = (preds == y).float().mean() - metrics = {"val_loss": loss, "val_acc": acc} - self.log_dict(metrics, on_step=False, on_epoch=True, prog_bar=True, logger=True) - return metrics + + # Update metrics + self.val_acc(preds, y) + self.precision_metric(preds, y) + self.recall(preds, y) + self.f1(preds, y) + # Convert logits to probabilities + probs = torch.softmax(logits, dim=1) + self.auroc(probs, y) + + self.log('val_loss', loss, prog_bar=True) + self.log('val_acc', self.val_acc, prog_bar=True) + self.log('val_precision', self.precision, prog_bar=True) + self.log('val_recall', self.recall, prog_bar=True) + self.log('val_f1', self.f1, prog_bar=True) + self.log('val_auroc', self.auroc, prog_bar=True) + return loss def configure_optimizers(self): return torch.optim.Adam(self.parameters(), lr=self.learning_rate) From f38f22d60534f068777ff50f1acc2fc597454a54 Mon Sep 17 00:00:00 2001 From: Jereml Date: Wed, 14 Jun 2023 22:59:04 +0200 Subject: [PATCH 9/9] Add testing model --- src/models/BirdClassifierResNet.py | 24 ++++++++++++++ src/models/spectrogram_dataset.py | 2 +- src/models/test_models.py | 52 ++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 src/models/test_models.py diff --git a/src/models/BirdClassifierResNet.py b/src/models/BirdClassifierResNet.py index 1b28f42..9010847 100644 --- a/src/models/BirdClassifierResNet.py +++ b/src/models/BirdClassifierResNet.py @@ -69,5 +69,29 @@ def validation_step(self, batch, batch_idx): self.log('val_auroc', self.auroc, prog_bar=True) return loss + def test_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.cross_entropy(logits, y) + preds = torch.argmax(logits, dim=1) + + # Update metrics + self.val_acc(preds, y) + self.precision_metric(preds, y) + self.recall(preds, y) + self.f1(preds, y) + # Convert logits to probabilities + probs = torch.softmax(logits, dim=1) + self.auroc(probs, y) + + self.log('test_loss', loss, prog_bar=True) + self.log('test_acc', self.val_acc, prog_bar=True) + self.log('test_precision', self.precision_metric, prog_bar=True) + self.log('test_recall', self.recall, prog_bar=True) + self.log('test_f1', self.f1, prog_bar=True) + self.log('test_auroc', self.auroc, prog_bar=True) + return loss + + def configure_optimizers(self): return torch.optim.Adam(self.parameters(), lr=self.learning_rate) diff --git a/src/models/spectrogram_dataset.py b/src/models/spectrogram_dataset.py index 52ce690..0944e98 100644 --- a/src/models/spectrogram_dataset.py +++ b/src/models/spectrogram_dataset.py @@ -54,7 +54,7 @@ def setup(self, stage=None): "train", "validation" ) if stage == "test" or stage is None: - self.test_dataset, _ = self._create_datasets("test") + self.test_dataset = self._create_datasets("test")[0] def _create_datasets(self, *splits): datasets = [] diff --git a/src/models/test_models.py b/src/models/test_models.py new file mode 100644 index 0000000..6ef7cd0 --- /dev/null +++ b/src/models/test_models.py @@ -0,0 +1,52 @@ +from pathlib import Path +import pytorch_lightning as pl +from BirdClassifierResNet import BirdClassifierResNet +from bird_spectrogram_classifier import BirdClassifier +from spectrogram_dataset import BirdSpectrogramDataModule +from pytorch_lightning.loggers import WandbLogger +import torch +import wandb + +root_dirs = { + "split1": "data/split_raw", + "split2": "data/split_aug", + "split3": "data/split_wrong", +} +num_classes = 7 +batch_size = 32 # single batch size +learning_rate = 0.001 # single learning rate + +models = [ + Path(f"split{num}_bird_classifier_rest.ckpt") for num in range(1,4) +] + + +def main(): + for split_name, root_dir in root_dirs.items(): + data_module = BirdSpectrogramDataModule(root_dir, batch_size=batch_size) + data_module.setup() + data_loders = [("train_subset", data_module.train_dataloader()), ("val_subset", data_module.val_dataloader()), ("test_subset", data_module.test_dataloader())] + for model_path in models: + # Initialize the model + model = BirdClassifierResNet.load_from_checkpoint(model_path, num_classes=num_classes, learning_rate=learning_rate) + + for subset_name, data_loder in data_loders: + # Set up the Weights & Biases logger + wandb_logger = WandbLogger( + project="bird-classification", + name=f"{split_name}_{model_path.name}_{subset_name}", + ) + + # Test the model + trainer = pl.Trainer(logger=wandb_logger) + trainer.test(model, dataloaders=data_loder) + + wandb_logger.experiment.config.update({ + "split_name": split_name, + "model_name": model_path.name, + "subset_name": subset_name + }) + wandb_logger.experiment.finish() + +if __name__ == "__main__": + main()