Point to latest nemo in dockerfile.

galvasr2/align/spark/align_cuda_decoder.py

@@ -145,7 +145,7 @@ def main(argv):
     mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf(
         "SC_PHYS_PAGES"
     )  # e.g. 4015976448
-    mem_gib = int((mem_bytes / (1024.0 ** 3)) * 0.9)
+    mem_gib = int((mem_bytes / (1024.0**3)) * 0.9)
     tar_jar = os.path.join(
         find_runfiles(), "__main__/galvasr2/spark/tar_spark_datasource.jar"
     )

galvasr2/align/spark/align_lib_test.py

@@ -128,7 +128,7 @@ def setUpClass(cls):
         mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf(
             "SC_PHYS_PAGES"
         )  # e.g. 4015976448
-        mem_gib = int((mem_bytes / (1024.0 ** 3)) * 0.9)
+        mem_gib = int((mem_bytes / (1024.0**3)) * 0.9)
         tar_jar = os.path.join(
             find_runfiles(), "__main__/galvasr2/spark/tar_spark_datasource.jar"
         )

model-training/Dockerfile

@@ -12,7 +12,8 @@ RUN apt-get update && \
 
 # Install NeMo from fork
 WORKDIR /workspace
-RUN git clone https://github.com/JFCeron/NeMo --single-branch --branch peoples-speech --depth=1
+RUN git clone https://github.com/galv/NeMo.git \
+    && cd NeMo && git checkout peoples-speech-head
 RUN pip install -e NeMo
 RUN pip install -r NeMo/requirements/requirements.txt
 RUN pip install -r NeMo/requirements/requirements_common.txt

model-training/conformer_ctc_char.yaml

@@ -3,22 +3,20 @@
 
 # You may find more detail on Conformer-CTC at `examples/asr/conf/conformer/conformer_ctc_bpe.yaml`
 
-name: ???
+name: "Conformer-CTC-Char"
 n_gpus: ???
 seed: 64
 
 model:
   sample_rate: 16000
   labels:  [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
             "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"]
-  log_prediction: false # enables logging sample predictions in the output during training
+  log_prediction: true # enables logging sample predictions in the output during training
   ctc_reduction: 'mean_batch'
-  use_cer: true
+  skip_nan_grad: false
 
   train_ds:
     manifest_filepath: ???
-    tarred_audio_filepaths: ???
-    is_tarred: true
     labels: ${model.labels}
     sample_rate: ${model.sample_rate}
     batch_size: 16 # you may increase batch_size if your memory allows
@@ -28,23 +26,28 @@ model:
     trim_silence: false
     max_duration: 16.7 # it is set for LibriSpeech, you may need to update it for your dataset
     min_duration: 0.1
-    max_cer: 0.13
-    tarred_shard_strategy: "scatter"
+    # tarred datasets
+    is_tarred: false
+    tarred_audio_filepaths: ???
+    shuffle_n: 2048
+    # bucketing params
+    bucketing_strategy: "synced_randomized"
+    bucketing_batch_size: null
 
   validation_ds:
     manifest_filepath: ???
-    sample_rate: ${model.sample_rate}
     labels: ${model.labels}
-    batch_size: 1
-    num_workers: 4
-    shuffle: False
+    sample_rate: ${model.sample_rate}
+    batch_size: 16 # you may increase batch_size if your memory allows
+    shuffle: false
+    num_workers: 8
     pin_memory: true
 
   test_ds:
     manifest_filepath: null
     labels: ${model.labels}
     sample_rate: ${model.sample_rate}
-    batch_size: 1 # you may increase batch_size if your memory allows
+    batch_size: 16 # you may increase batch_size if your memory allows
     shuffle: false
     num_workers: 8
     pin_memory: true
@@ -66,9 +69,9 @@ model:
 
   spec_augment:
     _target_: nemo.collections.asr.modules.SpectrogramAugmentation
-    freq_masks: 0 # set to zero to disable it
+    freq_masks: 2 # set to zero to disable it
     # you may use lower time_masks for smaller models to have a faster convergence
-    time_masks: 0 # set to zero to disable it
+    time_masks: 10 # set to zero to disable it
     freq_width: 27
     time_width: 0.05
 
@@ -77,7 +80,7 @@ model:
     feat_in: ${model.preprocessor.features}
     feat_out: -1 # you may set it if you need different output size other than the default d_model
     n_layers: 16
-    d_model: 320
+    d_model: 256
 
     # Sub-sampling params
     subsampling: striding # vggnet or striding, vggnet may give better results but needs more memory
@@ -98,6 +101,7 @@ model:
 
     # Convolution module's params
     conv_kernel_size: 31
+    conv_norm_type: 'batch_norm' # batch_norm or layer_norm
 
     ### regularization
     dropout: 0.1 # The dropout used in most of the Conformer Modules
@@ -107,18 +111,18 @@ model:
   decoder:
     _target_: nemo.collections.asr.modules.ConvASRDecoder
     feat_in: null
-    num_classes: 28
+    num_classes: -1
     vocabulary: ${model.labels}
 
   optim:
     name: adamw
-    lr: 0.1
+    lr: 2.0
     # optimizer arguments
     betas: [0.9, 0.98]
     # less necessity for weight_decay as we already have large augmentations with SpecAug
     # you may need weight_decay for large models, stable AMP training, small datasets, or when lower augmentations are used
     # weight decay of 0.0 with lr of 2.0 also works fine
-    weight_decay: 0.0
+    weight_decay: 1e-3
 
     # scheduler setup
     sched:
@@ -127,28 +131,28 @@ model:
       # scheduler config override
       warmup_steps: 10000
       warmup_ratio: null
-      min_lr: 1e-10
+      min_lr: 1e-6
 
 trainer:
-  gpus: ${n_gpus} # number of GPUs, -1 would use all available GPUs
+  devices: -1 # number of GPUs, -1 would use all available GPUs
   num_nodes: 1
-  max_epochs: 10
+  max_epochs: 1000
   max_steps: null # computed at runtime if not set
-  val_check_interval: 0.05 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
-  accelerator: ddp
+  val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
+  accelerator: auto
+  strategy: ddp
   accumulate_grad_batches: 1
   gradient_clip_val: 0.0
-  amp_level: O2 # O1/O2 for mixed precision
-  amp_backend: apex
-  precision: 16 # Should be set to 16 for O1 and O2 to enable the AMP.
-  log_every_n_steps: 1  # Interval of logging.
+  precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP.
+  log_every_n_steps: 10  # Interval of logging.
   progress_bar_refresh_rate: 10
   resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.
   num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
   check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs
   sync_batchnorm: true
-  checkpoint_callback: false  # Provided by exp_manager
+  enable_checkpointing: False  # Provided by exp_manager
   logger: false  # Provided by exp_manager
+  benchmark: false # needs to be false for models with variable-length speech input as it slows down training
 
 
 exp_manager:
@@ -160,14 +164,15 @@ exp_manager:
     # in case of multiple validation sets, first one is used
     monitor: "val_wer"
     mode: "min"
-    save_top_k: 3
+    save_top_k: 5
+    always_save_nemo: True # saves the checkpoints as nemo files instead of PTL checkpoints
 
   # you need to set these two to True to continue the training
-  # resume_if_exists: false
-  # resume_ignore_no_checkpoint: false
+  #resume_if_exists: false
+  #resume_ignore_no_checkpoint: false
 
   # You may use this section to create a W&B logger
-  create_wandb_logger: false
-  wandb_logger_kwargs:
-    name: null
-    project: null
+  #create_wandb_logger: false
+  #wandb_logger_kwargs:
+    #name: null
+    #project: null

model-training/conformer_ctc_char_tutorial.yaml

@@ -0,0 +1,178 @@
+# It contains the default values for training a Conformer-CTC ASR model, large size (~120M) with CTC loss and char-based vocabulary.
+# Char-based encoding may give lower accuracy than sub-word encoding (conformer_ctc_bpe.yaml).
+
+# You may find more detail on Conformer-CTC at `examples/asr/conf/conformer/conformer_ctc_bpe.yaml`
+
+name: "Conformer-CTC-Char"
+n_gpus: 1
+seed: 64
+
+model:
+  sample_rate: 16000
+  labels:  [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
+            "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"]
+  log_prediction: true # enables logging sample predictions in the output during training
+  ctc_reduction: 'mean_batch'
+  skip_nan_grad: false
+
+  train_ds:
+    manifest_filepath: /data/the-peoples-speech/cc-by-clean/nemo_format_manifest.jsonl
+    labels: ${model.labels}
+    sample_rate: ${model.sample_rate}
+    batch_size: 16 # you may increase batch_size if your memory allows
+    shuffle: true
+    num_workers: 8
+    pin_memory: true
+    trim_silence: false
+    max_duration: 16.7 # it is set for LibriSpeech, you may need to update it for your dataset
+    min_duration: 0.1
+    # tarred datasets
+    is_tarred: true
+    tarred_audio_filepaths: ['/data/the-peoples-speech/cc-by-clean/audios/clean_003112.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_006100.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_005107.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_001983.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_001354.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004500.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004167.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_006446.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_006863.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004353.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_010963.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_002015.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_000591.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004434.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004546.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_005451.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_010742.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_002227.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_002134.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_003505.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_005088.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004726.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_008553.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_000979.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_000243.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_011018.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_006069.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_001188.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004215.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_007987.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_000877.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_005041.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_006234.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004674.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_001663.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_009087.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_007260.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_007950.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_000344.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004077.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004305.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_007714.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_010054.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_010488.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004318.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_005287.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_004249.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_007623.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_006132.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_008651.tar', '/data/the-peoples-speech/cc-by-clean/audios/clean_009961.tar']
+    shuffle_n: 2048
+    # bucketing params
+    bucketing_strategy: "synced_randomized"
+    bucketing_batch_size: null
+
+  validation_ds:
+    manifest_filepath: "/data/librispeech/dev_clean.json"
+    labels: ${model.labels}
+    sample_rate: ${model.sample_rate}
+    batch_size: 16 # you may increase batch_size if your memory allows
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+
+  test_ds:
+    manifest_filepath: null
+    labels: ${model.labels}
+    sample_rate: ${model.sample_rate}
+    batch_size: 16 # you may increase batch_size if your memory allows
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+
+  preprocessor:
+    _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor
+    sample_rate: ${model.sample_rate}
+    normalize: "per_feature"
+    window_size: 0.025
+    window_stride: 0.01
+    window: "hann"
+    features: 80
+    n_fft: 512
+    log: true
+    frame_splicing: 1
+    dither: 0.00001
+    pad_to: 0
+    pad_value: 0.0
+
+  spec_augment:
+    _target_: nemo.collections.asr.modules.SpectrogramAugmentation
+    freq_masks: 2 # set to zero to disable it
+    # you may use lower time_masks for smaller models to have a faster convergence
+    time_masks: 10 # set to zero to disable it
+    freq_width: 27
+    time_width: 0.05
+
+  encoder:
+    _target_: nemo.collections.asr.modules.ConformerEncoder
+    feat_in: ${model.preprocessor.features}
+    feat_out: -1 # you may set it if you need different output size other than the default d_model
+    n_layers: 16
+    d_model: 256
+
+    # Sub-sampling params
+    subsampling: striding # vggnet or striding, vggnet may give better results but needs more memory
+    subsampling_factor: 4 # must be power of 2
+    subsampling_conv_channels: -1 # set to -1 to make it equal to the d_model
+
+    # Feed forward module's params
+    ff_expansion_factor: 4
+
+    # Multi-headed Attention Module's params
+    self_attention_model: rel_pos # rel_pos or abs_pos
+    n_heads: 8 # may need to be lower for smaller d_models
+    # [left, right] specifies the number of steps to be seen from left and right of each step in self-attention
+    att_context_size: [-1, -1] # -1 means unlimited context
+    xscaling: true # scales up the input embeddings by sqrt(d_model)
+    untie_biases: true # unties the biases of the TransformerXL layers
+    pos_emb_max_len: 5000
+
+    # Convolution module's params
+    conv_kernel_size: 31
+    conv_norm_type: 'batch_norm' # batch_norm or layer_norm
+
+    ### regularization
+    dropout: 0.1 # The dropout used in most of the Conformer Modules
+    dropout_emb: 0.0 # The dropout used for embeddings
+    dropout_att: 0.1 # The dropout for multi-headed attention modules
+
+  decoder:
+    _target_: nemo.collections.asr.modules.ConvASRDecoder
+    feat_in: null
+    num_classes: -1
+    vocabulary: ${model.labels}
+
+  optim:
+    name: adamw
+    lr: 2.0
+    # optimizer arguments
+    betas: [0.9, 0.98]
+    # less necessity for weight_decay as we already have large augmentations with SpecAug
+    # you may need weight_decay for large models, stable AMP training, small datasets, or when lower augmentations are used
+    # weight decay of 0.0 with lr of 2.0 also works fine
+    weight_decay: 1e-3
+
+    # scheduler setup
+    sched:
+      name: CosineAnnealing
+
+      # scheduler config override
+      warmup_steps: 1000
+      warmup_ratio: null
+      min_lr: 1e-6
+
+trainer:
+  devices: -1 # number of GPUs, -1 would use all available GPUs
+  num_nodes: 1
+  max_epochs: 1000
+  max_steps: null # computed at runtime if not set
+  val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
+  accelerator: auto
+  strategy: ddp
+  accumulate_grad_batches: 1
+  gradient_clip_val: 0.0
+  precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP.
+  log_every_n_steps: 10  # Interval of logging.
+  progress_bar_refresh_rate: 10
+  resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.
+  num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
+  check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs
+  sync_batchnorm: true
+  enable_checkpointing: False  # Provided by exp_manager
+  logger: false  # Provided by exp_manager
+  benchmark: false # needs to be false for models with variable-length speech input as it slows down training
+
+
+exp_manager:
+  exp_dir: /experiments
+  name: ${name}
+  create_tensorboard_logger: true
+  create_checkpoint_callback: true
+  checkpoint_callback_params:
+    # in case of multiple validation sets, first one is used
+    monitor: "val_wer"
+    mode: "min"
+    save_top_k: 5
+    always_save_nemo: True # saves the checkpoints as nemo files instead of PTL checkpoints
+
+  # you need to set these two to True to continue the training
+  #resume_if_exists: false
+  #resume_ignore_no_checkpoint: false
+
+  # You may use this section to create a W&B logger
+  #create_wandb_logger: false
+  #wandb_logger_kwargs:
+    #name: null
+    #project: null