mlcommons · jeussej · Nov 9, 2022 · Nov 14, 2022 · Nov 14, 2022 · Nov 15, 2022
@@ -18,7 +18,6 @@ def convert_and_filter_topk(args):
     with io.TextIOWrapper(
         io.BufferedWriter(gzip.open(data_lower, "w+")), encoding="utf-8"
     ) as file_out:
-
         # Open the input file either from input.txt or input.txt.gz
         _, file_extension = os.path.splitext(args.input_txt)
         if file_extension == ".gz":

@@ -15,7 +15,6 @@ def convert_and_filter_topk(output_dir, input_txt, top_k):
     with io.TextIOWrapper(
         io.BufferedWriter(gzip.open(data_lower, "w+")), encoding="utf-8"
     ) as file_out:
-
         # Open the input file either from input.txt or input.txt.gz
         _, file_extension = os.path.splitext(input_txt)
         if file_extension == ".gz":

@@ -259,7 +259,6 @@ def main(argv):
             ],
             unmount_cmd=["fusermount", "-u"],
         ) as temp_dir_name:
-
             posix_ctm_out_dir = re.sub(
                 r"^{0}".format(FLAGS.input_gcs_bucket), temp_dir_name, ctm_out_dir
             )

@@ -186,7 +186,8 @@ def __len__(self):
 class LimitingPool:
     """Limits unbound ahead-processing of multiprocessing.Pool's imap method
     before items get consumed by the iteration caller.
-    This prevents OOM issues in situations where items represent larger memory allocations."""
+    This prevents OOM issues in situations where items represent larger memory allocations.
+    """
 
     def __init__(self, processes=None, limit_factor=2, sleeping_for=0.1):
         self.processes = os.cpu_count() if processes is None else processes

@@ -2,6 +2,7 @@
 import re
 import sys
 
+
 # https://stackoverflow.com/a/45176191
 def find_runfiles():
     """Find the runfiles tree (useful when _not_ run from a zip file)"""

@@ -82,7 +82,6 @@ def run_inference(config):
 
 
 def get_dataset(config):
-
     logger.debug("Getting file paths")
     files, filenames = list_files(config["input_path"], config)
 
@@ -237,9 +236,7 @@ def download(url, path):
 
 
 def run_model_on_dataset(yamnet, classes, params, dataset, filenames, config):
-
     with jsonlines.open(config["output_path"], mode="w") as writer:
-
         for batch, filename in zip(dataset, filenames):
             logger.debug(filename)
             items = split_into_items(batch, config)
@@ -295,7 +292,6 @@ def print_results(writer, filename, results, yamnet_classes, index, config):
 
 
 def run_model_on_batch(yamnet, classes, params, pair):
-
     batch, sr = pair
 
     waveform = batch / 32768.0  # Convert to [-1.0, +1.0]
@@ -339,7 +335,6 @@ def config_path():
 
 
 def setup_logging(arguments):
-
     logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
 
     if arguments["verbose"]:

@@ -127,7 +127,6 @@ def config_path():
 
 
 def setup_logging(arguments):
-
     logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
 
     if arguments["verbose"]:

@@ -17,6 +17,7 @@
 
 from dataclasses import dataclass
 
+
 # The following hyperparameters (except patch_hop_seconds) were used to train YAMNet,
 # so expect some variability in performance if you change these. The patch hop can
 # be changed arbitrarily: a smaller hop should give you more patches from the same

@@ -108,7 +108,7 @@ def yamnet(features, params):
         (params.patch_frames, params.patch_bands, 1),
         input_shape=(params.patch_frames, params.patch_bands),
     )(features)
-    for (i, (layer_fun, kernel, stride, filters)) in enumerate(_YAMNET_LAYER_DEFS):
+    for i, (layer_fun, kernel, stride, filters) in enumerate(_YAMNET_LAYER_DEFS):
         net = layer_fun("layer{}".format(i + 1), kernel, stride, filters, params)(net)
     embeddings = layers.GlobalAveragePooling2D()(net)
     logits = layers.Dense(units=params.num_classes, use_bias=True)(embeddings)

@@ -133,7 +133,6 @@ def GetCombinedUttIndexRange(utt_index, utts, utt_durs, minimum_duration):
     cur_utt_dur = utt_durs[utts[utt_index]]
 
     while num_remaining_segments > 0:
-
         left_utt_dur = 0
         if left_index >= 0:
             left_utt_dur = utt_durs[utts[left_index]]
@@ -267,7 +266,6 @@ def CombineSegments(input_dir, output_dir, minimum_duration):
     speakers = spk2utt.keys()
     speakers.sort()
     for speaker in speakers:
-
         utts = spk2utt[speaker]  # this is an assignment of the reference
         # In WriteCombinedDirFiles the values of spk2utt will have the list
         # of combined utts which will be used as reference

@@ -530,7 +530,7 @@ def get_ctm_edits(
     # current_time is the end of the last ctm segment we processesed.
     current_time = ctm_array[0][0] if ctm_len > 0 else 0.0
 
-    for (ref_word, hyp_word, ref_prev_i, hyp_prev_i, ref_i, hyp_i) in alignment_output:
+    for ref_word, hyp_word, ref_prev_i, hyp_prev_i, ref_i, hyp_i in alignment_output:
         try:
             ctm_pos = hyp_prev_i
             # This is true because we cannot have errors at the end because

@@ -57,6 +57,7 @@
         )
     )
 
+
 # This processes one group of input lines; 'group_of_lines' is
 # an array of lines of input integerized text, e.g.
 # [ 'utt1 67 89 432', 'utt2 89 48 62' ]

@@ -41,7 +41,7 @@
 
 # Parse the prf records into dictionary,
 prf_dict = dict()
-for (f, c, t, e) in prf:
+for f, c, t, e in prf:
     t_pos = 0  # position in the 't' string,
     while t_pos < len(t):
         t1 = t[t_pos:].split(" ", 1)[0]  # get 1st token at 't_pos'

@@ -85,7 +85,7 @@
 # Build the targets,
 if o.conf_targets != "":
     with open(o.conf_targets, "w") as f:
-        for (utt, chan, beg, dur, wrd_id, conf, score_tag) in ctm:
+        for utt, chan, beg, dur, wrd_id, conf, score_tag in ctm:
             # Skip the words we don't know if being correct,
             if score_tag == "U":
                 continue
@@ -124,7 +124,7 @@
 
 # Build the input features,
 with open(o.conf_feats, "w") as f:
-    for (utt, chan, beg, dur, wrd_id, conf, score_tag) in ctm:
+    for utt, chan, beg, dur, wrd_id, conf, score_tag in ctm:
         # Build the key, same as previously,
         key = "%s^%s^%s^%s^%s,%s,%s" % (utt, chan, beg, dur, wrd_id, conf, score_tag)
 

@@ -169,7 +169,6 @@ def GetMean(depth_to_count):
 for phone, depths in sorted(
     phone_depth_counts.items(), key=lambda x: -sum(x[1].values())
 ):
-
     frequency_percentage = sum(depths.values()) * 100.0 / total_frames
     if frequency_percentage < args.frequency_cutoff_percentage:
         continue

@@ -146,7 +146,7 @@ def PruneProns(
     # have stats, we append them to the "stats" dict, with a zero count.
     for word, entry in stats.iteritems():
         prons_with_stats = set()
-        for (pron, count) in entry:
+        for pron, count in entry:
             prons_with_stats.add(pron)
         for pron in lexicon_g2p[word]:
             if pron not in prons_with_stats:

@@ -133,7 +133,7 @@ def Main():
         if phones not in prons[word]:
             prons[word].append(phones)
 
-    for (word, utt) in stats:
+    for word, utt in stats:
         count_sum = 0.0
         counts = dict()
         for phones in stats[(word, utt)]:

@@ -237,7 +237,7 @@ def OneEMIter(args, word, stats, prons, pron_probs, debug=False):
     for i in range(len(pron_probs)):
         pron_probs[i] = pron_probs[i] / s
     log_like = 0.0
-    for (utt, start_frame) in stats[word]:
+    for utt, start_frame in stats[word]:
         prob = []
         soft_counts = []
         for i in range(len(prons[word])):

@@ -218,7 +218,6 @@ def fill_nonlin_stats_table_with_regex_result(groups, gate_index, stats_table):
 
 
 def parse_progress_logs_for_nonlinearity_stats(exp_dir):
-
     """Parse progress logs for mean and std stats for non-linearities.
     e.g. for a line that is parsed from progress.*.log:
     exp/nnet3/lstm_self_repair_ld5_sp/log/progress.9.log:component name=Lstm3_i

@@ -651,7 +651,6 @@ def compute_train_cv_probabilities(
 
 
 def compute_progress(dir, iter, run_opts):
-
     prev_model = "{0}/{1}.mdl".format(dir, iter - 1)
     model = "{0}/{1}.mdl".format(dir, iter)
 

@@ -143,7 +143,6 @@ def get_successful_models(num_models, log_file_pattern, difference_threshold=1.0
 
 
 def get_average_nnet_model(dir, iter, nnets_list, run_opts, get_raw_nnet_from_am=True):
-
     next_iter = iter + 1
     if get_raw_nnet_from_am:
         out_model = """- \| nnet3-am-copy --set-raw-nnet=-  \
@@ -169,7 +168,6 @@ def get_average_nnet_model(dir, iter, nnets_list, run_opts, get_raw_nnet_from_am
 def get_best_nnet_model(
     dir, iter, best_model_index, run_opts, get_raw_nnet_from_am=True
 ):
-
     best_model = "{dir}/{next_iter}.{best_model_index}.raw".format(
         dir=dir, next_iter=iter + 1, best_model_index=best_model_index
     )
@@ -550,7 +548,6 @@ def verify_egs_dir(
 def compute_presoftmax_prior_scale(
     dir, alidir, num_jobs, run_opts, presoftmax_prior_scale_power=-0.25
 ):
-
     # getting the raw pdf count
     common_lib.execute_command(
         """{command} JOB=1:{num_jobs} {dir}/log/acc_pdf.JOB.log \
@@ -707,7 +704,6 @@ def get_learning_rate(
 def should_do_shrinkage(
     iter, model_file, shrink_saturation_threshold, get_raw_nnet_from_am=True
 ):
-
     if iter == 0:
         return True
 

@@ -33,7 +33,6 @@ def generate_egs(
     egs_opts=None,
     cmvn_opts=None,
 ):
-
     """Wrapper for calling steps/nnet3/get_egs.sh
 
     Generates targets from alignment directory 'alidir', which contains

@@ -703,7 +703,6 @@ def get_realign_iters(realign_times, num_iters, num_jobs_initial, num_jobs_final
 
 
 def align(dir, data, lang, run_opts, iter=None, online_ivector_dir=None):
-
     alidir = "{dir}/ali{ali_suffix}".format(
         dir=dir, ali_suffix="_iter_{0}".format(iter) if iter is not None else ""
     )
Original file line number	Diff line number	Diff line change
Expand Up		@@ -127,7 +127,6 @@ def config_path():


		def setup_logging(arguments):

		logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"

		if arguments["verbose"]:
Expand Down
-Original file line number
+Diff line change
@@ Expand Up / @@ -57,6 +57,7 @@ @@
             )
         )
     # This processes one group of input lines; 'group_of_lines' is
     # an array of lines of input integerized text, e.g.
     # [ 'utt1 67 89 432', 'utt2 89 48 62' ]
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
Expand Up		@@ -651,7 +651,6 @@ def compute_train_cv_probabilities(


		def compute_progress(dir, iter, run_opts):

		prev_model = "{0}/{1}.mdl".format(dir, iter - 1)
		model = "{0}/{1}.mdl".format(dir, iter)

Expand Down