From 8c0dd7f24fd8a14c893b0520d25ce95002672e7d Mon Sep 17 00:00:00 2001 From: greg-landing Date: Fri, 16 Apr 2021 09:35:38 -0700 Subject: [PATCH 1/3] ADD: Platform services, --- platform/micro-services/cloud-hal/README.md | 12 + .../cloud-hal/source/cloud_hal/__init__.py | 3 + .../source/cloud_hal/api/__init__.py | 3 + .../cloud-hal/source/cloud_hal/api/is_path.py | 14 + .../source/cloud_hal/storage/__init__.py | 3 + .../source/cloud_hal/storage/client.py | 8 + .../cloud_hal/storage/client_factory.py | 19 + .../storage/local_filesystem_client.py | 25 ++ .../command-line-interface/README.md | 13 + .../config/default.yaml | 14 + .../source/cli/peoples-speech.py | 79 ++++ platform/micro-services/data-book/README.md | 19 + .../data-book/hindi-databook.csv | 42 ++ .../data-book/marathi-databook.csv | 40 ++ .../data-book/source/data_book/__init__.py | 0 .../source/data_book/api/__init__.py | 4 + .../source/data_book/api/import_data_book.py | 5 + .../source/data_book/api/load_databook.py | 19 + .../command-line-interface/data-book.py | 49 +++ .../data_book/importer/fast_text_importer.py | 205 +++++++++ .../data_book/importer/importer_factory.py | 16 + platform/micro-services/data-export/README.md | 13 + .../data-export/source/add-clean-text.py | 86 ++++ .../source/add-duration-metadata.py | 233 ++++++++++ .../source/add-google-speech-transcript.py | 120 ++++++ .../source/add-librispeech-metadata.py | 121 ++++++ .../source/convert-audio-format.py | 278 ++++++++++++ .../data-export/source/convert-cc-search.py | 399 ++++++++++++++++++ .../source/convert-common-voice.py | 122 ++++++ .../source/convert-dipco-dataset-format.py | 187 ++++++++ .../data-export/source/convert-librispeech.py | 108 +++++ .../source/convert-librivox-google-cloud.py | 392 +++++++++++++++++ .../data-export/source/convert-librivox.py | 253 +++++++++++ .../source/convert-timit-dataset-format.py | 109 +++++ .../data-export/source/convert-voicery.py | 152 +++++++ .../data-export/source/convert-warc-to-csv.py | 110 +++++ .../data-export/source/count-duration.py | 0 .../source/data_export/__init__.py | 5 + .../source/data_export/api/save_dataset.py | 9 + .../source/make-librivox-train-test-split.py | 123 ++++++ .../make-peoples-speech-train-test-splits.py | 215 ++++++++++ .../source/make-tar-google-cloud.py | 186 ++++++++ .../micro-services/data-management/README.md | 13 + .../micro-services/deep-learning/README.md | 23 + .../micro-services/forced-alignment/README.md | 10 + .../source/forced_alignment/api/__init__.py | 6 + .../forced_alignment/api/align_dataset.py | 16 + .../ingestion/source/__init__.py | 3 + .../source/ingestion/api/__init__.py | 3 + .../ingestion/api/ingest_search_hits.py | 21 + .../source/peoples_speech/__init__.py | 0 .../peoples_speech/data_book/__init__.py | 3 + .../peoples_speech/data_export/__init__.py | 3 + .../peoples_speech/task_manager/__init__.py | 2 + platform/micro-services/quality/README.md | 11 + .../quality/source/quality/api/__init__.py | 4 + .../source/quality/api/clean_dataset.py | 5 + platform/micro-services/search/README.md | 10 + .../search/source/search/__init__.py | 0 .../search/source/search/api/__init__.py | 3 + .../source/search/api/search_for_data_book.py | 5 + .../command-line-interface/speech-search.py | 67 +++ .../engine/existing_dataset_search_engine.py | 109 +++++ .../search/engine/search_engine_factory.py | 14 + .../search/source/search/util/__init__.py | 6 + .../source/search/util/load_databook.py | 10 + .../search/source/search/util/save_dataset.py | 11 + .../search/test/marathi-databook.csv | 16 + .../micro-services/search/test/mr/the.mp3 | Bin 0 -> 2112 bytes .../\340\244\205\340\244\270\340\245\207.mp3" | Bin 0 -> 2688 bytes .../\340\244\206\340\244\243\340\244\277.mp3" | Bin 0 -> 2400 bytes .../\340\244\206\340\244\271\340\245\207.mp3" | Bin 0 -> 2688 bytes ...6\340\244\271\340\245\207\340\244\244.mp3" | Bin 0 -> 2880 bytes .../test/mr/\340\244\217\340\244\225.mp3" | Bin 0 -> 2016 bytes .../test/mr/\340\244\244\340\244\260.mp3" | Bin 0 -> 2016 bytes .../test/mr/\340\244\244\340\245\207.mp3" | Bin 0 -> 2304 bytes ...0\340\244\276\340\244\271\340\245\200.mp3" | Bin 0 -> 2976 bytes .../test/mr/\340\244\252\340\244\243.mp3" | Bin 0 -> 2016 bytes .../test/mr/\340\244\257\340\244\276.mp3" | Bin 0 -> 2112 bytes .../search/test/mr/\340\244\265.mp3" | Bin 0 -> 2016 bytes .../test/mr/\340\244\271\340\244\276.mp3" | Bin 0 -> 2016 bytes .../test/mr/\340\244\271\340\245\200.mp3" | Bin 0 -> 2112 bytes .../test/mr/\340\244\271\340\245\207.mp3" | Bin 0 -> 2112 bytes ...1\340\245\213\340\244\244\340\245\207.mp3" | Bin 0 -> 2784 bytes .../search/test/test-dataset.csv | 16 + .../micro-services/task-manager/README.md | 12 + .../source/task_manager/__init__.py | 0 .../task_manager/engine/engine_factory.py | 15 + .../task_manager/engine/local_engine.py | 11 + .../source/task_manager/pipelines/__init__.py | 0 .../task_manager/pipelines/make_dataset.py | 32 ++ platform/micro-services/website/README.md | 10 + .../micro-services/website/requirements.txt | 3 + .../website/source/config/default.json | 10 + .../website/source/flask/app.py | 22 + .../website/source/flask/gcloud_sign_url.py | 59 +++ .../website/source/react/App.css | 38 ++ .../website/source/react/App.js | 76 ++++ .../website/source/react/App.test.js | 8 + .../website/source/react/getSignUrl.js | 5 + .../source/react/getTrainingSetPath.js | 9 + .../website/source/react/index.css | 13 + .../website/source/react/index.js | 17 + .../website/source/react/logo.png | Bin 0 -> 23989 bytes .../website/source/react/logo.svg | 1 + .../website/source/react/reportWebVitals.js | 13 + .../website/source/react/setupTests.js | 5 + platform/micro-services/website/start-dev | 66 +++ platform/peoples-speech | 51 +++ .../gcloud-ubuntu-20.4/provision.sh | 0 platform/requirements.txt | 6 + 111 files changed, 4685 insertions(+) create mode 100644 platform/micro-services/cloud-hal/README.md create mode 100644 platform/micro-services/cloud-hal/source/cloud_hal/__init__.py create mode 100644 platform/micro-services/cloud-hal/source/cloud_hal/api/__init__.py create mode 100644 platform/micro-services/cloud-hal/source/cloud_hal/api/is_path.py create mode 100644 platform/micro-services/cloud-hal/source/cloud_hal/storage/__init__.py create mode 100644 platform/micro-services/cloud-hal/source/cloud_hal/storage/client.py create mode 100644 platform/micro-services/cloud-hal/source/cloud_hal/storage/client_factory.py create mode 100644 platform/micro-services/cloud-hal/source/cloud_hal/storage/local_filesystem_client.py create mode 100644 platform/micro-services/command-line-interface/README.md create mode 100644 platform/micro-services/command-line-interface/config/default.yaml create mode 100644 platform/micro-services/command-line-interface/source/cli/peoples-speech.py create mode 100644 platform/micro-services/data-book/README.md create mode 100644 platform/micro-services/data-book/hindi-databook.csv create mode 100644 platform/micro-services/data-book/marathi-databook.csv create mode 100644 platform/micro-services/data-book/source/data_book/__init__.py create mode 100644 platform/micro-services/data-book/source/data_book/api/__init__.py create mode 100644 platform/micro-services/data-book/source/data_book/api/import_data_book.py create mode 100644 platform/micro-services/data-book/source/data_book/api/load_databook.py create mode 100644 platform/micro-services/data-book/source/data_book/command-line-interface/data-book.py create mode 100644 platform/micro-services/data-book/source/data_book/importer/fast_text_importer.py create mode 100644 platform/micro-services/data-book/source/data_book/importer/importer_factory.py create mode 100644 platform/micro-services/data-export/README.md create mode 100644 platform/micro-services/data-export/source/add-clean-text.py create mode 100644 platform/micro-services/data-export/source/add-duration-metadata.py create mode 100644 platform/micro-services/data-export/source/add-google-speech-transcript.py create mode 100644 platform/micro-services/data-export/source/add-librispeech-metadata.py create mode 100644 platform/micro-services/data-export/source/convert-audio-format.py create mode 100644 platform/micro-services/data-export/source/convert-cc-search.py create mode 100644 platform/micro-services/data-export/source/convert-common-voice.py create mode 100644 platform/micro-services/data-export/source/convert-dipco-dataset-format.py create mode 100644 platform/micro-services/data-export/source/convert-librispeech.py create mode 100644 platform/micro-services/data-export/source/convert-librivox-google-cloud.py create mode 100644 platform/micro-services/data-export/source/convert-librivox.py create mode 100644 platform/micro-services/data-export/source/convert-timit-dataset-format.py create mode 100644 platform/micro-services/data-export/source/convert-voicery.py create mode 100644 platform/micro-services/data-export/source/convert-warc-to-csv.py create mode 100644 platform/micro-services/data-export/source/count-duration.py create mode 100644 platform/micro-services/data-export/source/data_export/__init__.py create mode 100644 platform/micro-services/data-export/source/data_export/api/save_dataset.py create mode 100644 platform/micro-services/data-export/source/make-librivox-train-test-split.py create mode 100644 platform/micro-services/data-export/source/make-peoples-speech-train-test-splits.py create mode 100644 platform/micro-services/data-export/source/make-tar-google-cloud.py create mode 100644 platform/micro-services/data-management/README.md create mode 100644 platform/micro-services/deep-learning/README.md create mode 100644 platform/micro-services/forced-alignment/README.md create mode 100644 platform/micro-services/forced-alignment/source/forced_alignment/api/__init__.py create mode 100644 platform/micro-services/forced-alignment/source/forced_alignment/api/align_dataset.py create mode 100644 platform/micro-services/ingestion/source/__init__.py create mode 100644 platform/micro-services/ingestion/source/ingestion/api/__init__.py create mode 100644 platform/micro-services/ingestion/source/ingestion/api/ingest_search_hits.py create mode 100644 platform/micro-services/peoples-speech/source/peoples_speech/__init__.py create mode 100644 platform/micro-services/peoples-speech/source/peoples_speech/data_book/__init__.py create mode 100644 platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py create mode 100644 platform/micro-services/peoples-speech/source/peoples_speech/task_manager/__init__.py create mode 100644 platform/micro-services/quality/README.md create mode 100644 platform/micro-services/quality/source/quality/api/__init__.py create mode 100644 platform/micro-services/quality/source/quality/api/clean_dataset.py create mode 100644 platform/micro-services/search/README.md create mode 100644 platform/micro-services/search/source/search/__init__.py create mode 100644 platform/micro-services/search/source/search/api/__init__.py create mode 100644 platform/micro-services/search/source/search/api/search_for_data_book.py create mode 100644 platform/micro-services/search/source/search/command-line-interface/speech-search.py create mode 100644 platform/micro-services/search/source/search/engine/existing_dataset_search_engine.py create mode 100644 platform/micro-services/search/source/search/engine/search_engine_factory.py create mode 100644 platform/micro-services/search/source/search/util/__init__.py create mode 100644 platform/micro-services/search/source/search/util/load_databook.py create mode 100644 platform/micro-services/search/source/search/util/save_dataset.py create mode 100644 platform/micro-services/search/test/marathi-databook.csv create mode 100644 platform/micro-services/search/test/mr/the.mp3 create mode 100644 "platform/micro-services/search/test/mr/\340\244\205\340\244\270\340\245\207.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\206\340\244\243\340\244\277.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\206\340\244\271\340\245\207.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\206\340\244\271\340\245\207\340\244\244.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\217\340\244\225.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\244\340\244\260.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\244\340\245\207.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\250\340\244\276\340\244\271\340\245\200.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\252\340\244\243.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\257\340\244\276.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\265.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\271\340\244\276.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\271\340\245\200.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\271\340\245\207.mp3" create mode 100644 "platform/micro-services/search/test/mr/\340\244\271\340\245\213\340\244\244\340\245\207.mp3" create mode 100644 platform/micro-services/search/test/test-dataset.csv create mode 100644 platform/micro-services/task-manager/README.md create mode 100644 platform/micro-services/task-manager/source/task_manager/__init__.py create mode 100644 platform/micro-services/task-manager/source/task_manager/engine/engine_factory.py create mode 100644 platform/micro-services/task-manager/source/task_manager/engine/local_engine.py create mode 100644 platform/micro-services/task-manager/source/task_manager/pipelines/__init__.py create mode 100644 platform/micro-services/task-manager/source/task_manager/pipelines/make_dataset.py create mode 100644 platform/micro-services/website/README.md create mode 100644 platform/micro-services/website/requirements.txt create mode 100644 platform/micro-services/website/source/config/default.json create mode 100644 platform/micro-services/website/source/flask/app.py create mode 100644 platform/micro-services/website/source/flask/gcloud_sign_url.py create mode 100644 platform/micro-services/website/source/react/App.css create mode 100644 platform/micro-services/website/source/react/App.js create mode 100644 platform/micro-services/website/source/react/App.test.js create mode 100644 platform/micro-services/website/source/react/getSignUrl.js create mode 100644 platform/micro-services/website/source/react/getTrainingSetPath.js create mode 100644 platform/micro-services/website/source/react/index.css create mode 100644 platform/micro-services/website/source/react/index.js create mode 100644 platform/micro-services/website/source/react/logo.png create mode 100644 platform/micro-services/website/source/react/logo.svg create mode 100644 platform/micro-services/website/source/react/reportWebVitals.js create mode 100644 platform/micro-services/website/source/react/setupTests.js create mode 100755 platform/micro-services/website/start-dev create mode 100755 platform/peoples-speech create mode 100644 platform/provisioning/gcloud-ubuntu-20.4/provision.sh create mode 100644 platform/requirements.txt diff --git a/platform/micro-services/cloud-hal/README.md b/platform/micro-services/cloud-hal/README.md new file mode 100644 index 00000000..30c52b2a --- /dev/null +++ b/platform/micro-services/cloud-hal/README.md @@ -0,0 +1,12 @@ + +# Cloud HAL + +Hardware abstraction layer for cloud platforms + +# Supported platforms + +* Local unix environment +* Google cloud + + + diff --git a/platform/micro-services/cloud-hal/source/cloud_hal/__init__.py b/platform/micro-services/cloud-hal/source/cloud_hal/__init__.py new file mode 100644 index 00000000..b7263844 --- /dev/null +++ b/platform/micro-services/cloud-hal/source/cloud_hal/__init__.py @@ -0,0 +1,3 @@ + +from cloud_hal.storage import client + diff --git a/platform/micro-services/cloud-hal/source/cloud_hal/api/__init__.py b/platform/micro-services/cloud-hal/source/cloud_hal/api/__init__.py new file mode 100644 index 00000000..2cec8bb6 --- /dev/null +++ b/platform/micro-services/cloud-hal/source/cloud_hal/api/__init__.py @@ -0,0 +1,3 @@ + +from cloud_hal.api.is_path import is_path + diff --git a/platform/micro-services/cloud-hal/source/cloud_hal/api/is_path.py b/platform/micro-services/cloud-hal/source/cloud_hal/api/is_path.py new file mode 100644 index 00000000..4d6bfd0d --- /dev/null +++ b/platform/micro-services/cloud-hal/source/cloud_hal/api/is_path.py @@ -0,0 +1,14 @@ +import os + +def is_path(path): + if os.path.exists(path): + return True + + if path.startswith("s3://"): + return True + + if path.startswith("gs://"): + return True + + return False + diff --git a/platform/micro-services/cloud-hal/source/cloud_hal/storage/__init__.py b/platform/micro-services/cloud-hal/source/cloud_hal/storage/__init__.py new file mode 100644 index 00000000..b6c02778 --- /dev/null +++ b/platform/micro-services/cloud-hal/source/cloud_hal/storage/__init__.py @@ -0,0 +1,3 @@ + +from cloud_hal.storage.client import client + diff --git a/platform/micro-services/cloud-hal/source/cloud_hal/storage/client.py b/platform/micro-services/cloud-hal/source/cloud_hal/storage/client.py new file mode 100644 index 00000000..8f00dcc9 --- /dev/null +++ b/platform/micro-services/cloud-hal/source/cloud_hal/storage/client.py @@ -0,0 +1,8 @@ + +from cloud_hal.storage.client_factory import ClientFactory + +def client(name="local", config={}): + return ClientFactory(name, config).create() + + + diff --git a/platform/micro-services/cloud-hal/source/cloud_hal/storage/client_factory.py b/platform/micro-services/cloud-hal/source/cloud_hal/storage/client_factory.py new file mode 100644 index 00000000..a6b83b22 --- /dev/null +++ b/platform/micro-services/cloud-hal/source/cloud_hal/storage/client_factory.py @@ -0,0 +1,19 @@ + +from cloud_hal.storage.local_filesystem_client import LocalFilesystemClient + +class ClientFactory: + def __init__(self, name, config): + self.name = name + self.config = config + + if len(name) == 0: + self.name = self.config["cloud_hal"]["storage"]["client"] + + def create(self): + if self.name == "local": + return LocalFilesystemClient(self.config) + + assert False, "Unknown client name " + self.name + + + diff --git a/platform/micro-services/cloud-hal/source/cloud_hal/storage/local_filesystem_client.py b/platform/micro-services/cloud-hal/source/cloud_hal/storage/local_filesystem_client.py new file mode 100644 index 00000000..96ce2b29 --- /dev/null +++ b/platform/micro-services/cloud-hal/source/cloud_hal/storage/local_filesystem_client.py @@ -0,0 +1,25 @@ + +import os +import shutil + +class LocalFilesystemClient: + def __init__(self, config): + self.config = config + + self.root = "/tmp" + + if "cloud_hal" in config: + if "storage" in config["cloud_hal"]: + if "root" in config["cloud_hal"]["storage"]: + self.root = config["cloud_hal"]["storage"]["root"] + + def exists(self, path): + return os.path.exists(path) + + def get_root(self): + return self.root + + def copy(self, src, dst): + shutil.copyfile(src, dst) + + diff --git a/platform/micro-services/command-line-interface/README.md b/platform/micro-services/command-line-interface/README.md new file mode 100644 index 00000000..5cb6a9a1 --- /dev/null +++ b/platform/micro-services/command-line-interface/README.md @@ -0,0 +1,13 @@ + +# peoples speech command line interface + +A command line interface to the peoples speech dataset services. + +# Usage + +``` + peoples-speech data-book-name dataset-name +``` + + + diff --git a/platform/micro-services/command-line-interface/config/default.yaml b/platform/micro-services/command-line-interface/config/default.yaml new file mode 100644 index 00000000..a91f313d --- /dev/null +++ b/platform/micro-services/command-line-interface/config/default.yaml @@ -0,0 +1,14 @@ + +task_manager: + engine: LocalEngine + +search: + engine: ExistingDatasetSearchEngine + dataset_paths: "test" + samples_per_page: 10 + +forced_alignment: + max_duration_seconds: 15 + + + diff --git a/platform/micro-services/command-line-interface/source/cli/peoples-speech.py b/platform/micro-services/command-line-interface/source/cli/peoples-speech.py new file mode 100644 index 00000000..ab16d058 --- /dev/null +++ b/platform/micro-services/command-line-interface/source/cli/peoples-speech.py @@ -0,0 +1,79 @@ + +import os + +from argparse import ArgumentParser + +import peoples_speech.data_book +import peoples_speech.data_export +import peoples_speech.task_manager + +import config + +import logging + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("The MLCommons data engineering framework.") + + parser.add_argument("-i", "--data-book-path", default=sample_databook_path(), help="Path to data book to generate a dataset for.") + parser.add_argument("-o", "--output-dataset-path", default="", help="The path to save the new dataset.") + parser.add_argument("-c", "--config-file-path", default=".csv", help="The path to save the new dataset.") + parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") + parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") + + arguments = vars(parser.parse_args()) + + config = setup_config(arguments) + + setup_logging(config) + + logger.debug("Full config: " + str(config)) + + data_book = peoples_speech.data_book.load_databook(config["data_book_path"]) + + dataset = peoples_speech.task_manager.make_dataset(config, data_book) + + peoples_speech.data_export.save_dataset(config, dataset) + +def sample_databook_path(): + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "search", "test", "marathi-databook.csv") + +def setup_config(dictionary): + return config.ConfigurationSet( + config.config_from_env(prefix="MLCOMMONS"), + config.config_from_yaml(config_path(), read_from_file=True), + config.config_from_dict(dictionary), + ) + +def config_path(): + home = os.path.expanduser("~") + home_config_path = os.path.join(home, ".mlcommons", "config.yaml") + if os.path.exists(home_config_path): + return home_config_path + + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "config", "default.yaml") + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + +if __name__ == "__main__": + main() + diff --git a/platform/micro-services/data-book/README.md b/platform/micro-services/data-book/README.md new file mode 100644 index 00000000..b27f579e --- /dev/null +++ b/platform/micro-services/data-book/README.md @@ -0,0 +1,19 @@ + +# Data book + +Human editable specification of a speech dataset. + +# Usage + +``` + peoples-speech data-book import fasttext marathi data-book-name +``` + +# Data format + +Data books are a set of pages. + +Pages are a pair: + * unicode text + * audio + diff --git a/platform/micro-services/data-book/hindi-databook.csv b/platform/micro-services/data-book/hindi-databook.csv new file mode 100644 index 00000000..82472df2 --- /dev/null +++ b/platform/micro-services/data-book/hindi-databook.csv @@ -0,0 +1,42 @@ +के,/tmp/cache/data_book/fasttext/hi/के.mp3 +।,/tmp/cache/data_book/fasttext/hi/।.mp3 +है,/tmp/cache/data_book/fasttext/hi/है.mp3 +में,/tmp/cache/data_book/fasttext/hi/में.mp3 +',/tmp/cache/data_book/fasttext/hi/'.mp3 +की,/tmp/cache/data_book/fasttext/hi/की.mp3 +का,/tmp/cache/data_book/fasttext/hi/का.mp3 +से,/tmp/cache/data_book/fasttext/hi/से.mp3 +-,/tmp/cache/data_book/fasttext/hi/-.mp3 +और,/tmp/cache/data_book/fasttext/hi/और.mp3 +को,/tmp/cache/data_book/fasttext/hi/को.mp3 +हैं,/tmp/cache/data_book/fasttext/hi/हैं.mp3 +>,/tmp/cache/data_book/fasttext/hi/>.mp3 +पर,/tmp/cache/data_book/fasttext/hi/पर.mp3 +एक,/tmp/cache/data_book/fasttext/hi/एक.mp3 +भी,/tmp/cache/data_book/fasttext/hi/भी.mp3 +ने,/tmp/cache/data_book/fasttext/hi/ने.mp3 +लिए,/tmp/cache/data_book/fasttext/hi/लिए.mp3 +कि,/tmp/cache/data_book/fasttext/hi/कि.mp3 +किया,/tmp/cache/data_book/fasttext/hi/किया.mp3 +था,/tmp/cache/data_book/fasttext/hi/था.mp3 +नहीं,/tmp/cache/data_book/fasttext/hi/नहीं.mp3 +ही,/tmp/cache/data_book/fasttext/hi/ही.mp3 +यह,/tmp/cache/data_book/fasttext/hi/यह.mp3 +इस,/tmp/cache/data_book/fasttext/hi/इस.mp3 +कर,/tmp/cache/data_book/fasttext/hi/कर.mp3 +हो,/tmp/cache/data_book/fasttext/hi/हो.mp3 +गया,/tmp/cache/data_book/fasttext/hi/गया.mp3 +तो,/tmp/cache/data_book/fasttext/hi/तो.mp3 +},/tmp/cache/data_book/fasttext/hi/}.mp3 +’,/tmp/cache/data_book/fasttext/hi/’.mp3 +‘,/tmp/cache/data_book/fasttext/hi/‘.mp3 +किस,/tmp/cache/data_book/fasttext/hi/किस.mp3 +करने,/tmp/cache/data_book/fasttext/hi/करने.mp3 +भारत,/tmp/cache/data_book/fasttext/hi/भारत.mp3 +जाता,/tmp/cache/data_book/fasttext/hi/जाता.mp3 +जो,/tmp/cache/data_book/fasttext/hi/जो.mp3 +साथ,/tmp/cache/data_book/fasttext/hi/साथ.mp3 +अपने,/tmp/cache/data_book/fasttext/hi/अपने.mp3 +या,/tmp/cache/data_book/fasttext/hi/या.mp3 +थी,/tmp/cache/data_book/fasttext/hi/थी.mp3 +द्वारा,/tmp/cache/data_book/fasttext/hi/द्वारा.mp3 diff --git a/platform/micro-services/data-book/marathi-databook.csv b/platform/micro-services/data-book/marathi-databook.csv new file mode 100644 index 00000000..441e0bb2 --- /dev/null +++ b/platform/micro-services/data-book/marathi-databook.csv @@ -0,0 +1,40 @@ +',/tmp/cache/data_book/fasttext/mr/'.mp3 +-,/tmp/cache/data_book/fasttext/mr/-.mp3 +आहे,/tmp/cache/data_book/fasttext/mr/आहे.mp3 +आणि,/tmp/cache/data_book/fasttext/mr/आणि.mp3 +},/tmp/cache/data_book/fasttext/mr/}.mp3 +या,/tmp/cache/data_book/fasttext/mr/या.mp3 +व,/tmp/cache/data_book/fasttext/mr/व.mp3 +हे,/tmp/cache/data_book/fasttext/mr/हे.mp3 +एक,/tmp/cache/data_book/fasttext/mr/एक.mp3 +हा,/tmp/cache/data_book/fasttext/mr/हा.mp3 +नाही,/tmp/cache/data_book/fasttext/mr/नाही.mp3 +ते,/tmp/cache/data_book/fasttext/mr/ते.mp3 +आहेत,/tmp/cache/data_book/fasttext/mr/आहेत.mp3 +होते,/tmp/cache/data_book/fasttext/mr/होते.mp3 +तर,/tmp/cache/data_book/fasttext/mr/तर.mp3 +the,/tmp/cache/data_book/fasttext/mr/the.mp3 +ही,/tmp/cache/data_book/fasttext/mr/ही.mp3 +#,/tmp/cache/data_book/fasttext/mr/#.mp3 +�,/tmp/cache/data_book/fasttext/mr/�.mp3 +पण,/tmp/cache/data_book/fasttext/mr/पण.mp3 +असे,/tmp/cache/data_book/fasttext/mr/असे.mp3 +"""","/tmp/cache/data_book/fasttext/mr/"".mp3" +काही,/tmp/cache/data_book/fasttext/mr/काही.mp3 +त्या,/tmp/cache/data_book/fasttext/mr/त्या.mp3 +’,/tmp/cache/data_book/fasttext/mr/’.mp3 +मी,/tmp/cache/data_book/fasttext/mr/मी.mp3 +of,/tmp/cache/data_book/fasttext/mr/of.mp3 +होता,/tmp/cache/data_book/fasttext/mr/होता.mp3 +इ.स,/tmp/cache/data_book/fasttext/mr/इ.स.mp3 +तो,/tmp/cache/data_book/fasttext/mr/तो.mp3 +होती,/tmp/cache/data_book/fasttext/mr/होती.mp3 +मराठी,/tmp/cache/data_book/fasttext/mr/मराठी.mp3 +|,/tmp/cache/data_book/fasttext/mr/|.mp3 +की,/tmp/cache/data_book/fasttext/mr/की.mp3 +केले,/tmp/cache/data_book/fasttext/mr/केले.mp3 +केली,/tmp/cache/data_book/fasttext/mr/केली.mp3 +to,/tmp/cache/data_book/fasttext/mr/to.mp3 +म्हणून,/tmp/cache/data_book/fasttext/mr/म्हणून.mp3 +‘,/tmp/cache/data_book/fasttext/mr/‘.mp3 +यांनी,/tmp/cache/data_book/fasttext/mr/यांनी.mp3 diff --git a/platform/micro-services/data-book/source/data_book/__init__.py b/platform/micro-services/data-book/source/data_book/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/platform/micro-services/data-book/source/data_book/api/__init__.py b/platform/micro-services/data-book/source/data_book/api/__init__.py new file mode 100644 index 00000000..b2949fef --- /dev/null +++ b/platform/micro-services/data-book/source/data_book/api/__init__.py @@ -0,0 +1,4 @@ + +from data_book.api.import_data_book import import_data_book + + diff --git a/platform/micro-services/data-book/source/data_book/api/import_data_book.py b/platform/micro-services/data-book/source/data_book/api/import_data_book.py new file mode 100644 index 00000000..d5f934cb --- /dev/null +++ b/platform/micro-services/data-book/source/data_book/api/import_data_book.py @@ -0,0 +1,5 @@ + +from data_book.importer.importer_factory import ImporterFactory + +def import_data_book(config): + ImporterFactory(config).create().import_data_book() diff --git a/platform/micro-services/data-book/source/data_book/api/load_databook.py b/platform/micro-services/data-book/source/data_book/api/load_databook.py new file mode 100644 index 00000000..66614d8f --- /dev/null +++ b/platform/micro-services/data-book/source/data_book/api/load_databook.py @@ -0,0 +1,19 @@ +import csv +import os + +def load_databook(path): + with open(path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + return [check_row(path, row) for row in reader] + +def check_row(data_book_path, row): + label, path = row + + if not os.path.isabs(path): + path = os.path.join(os.path.dirname(data_book_path), path) + + assert os.path.exists(path), "Path does not exist: " + path + + return label, path + + diff --git a/platform/micro-services/data-book/source/data_book/command-line-interface/data-book.py b/platform/micro-services/data-book/source/data_book/command-line-interface/data-book.py new file mode 100644 index 00000000..fc232b7f --- /dev/null +++ b/platform/micro-services/data-book/source/data_book/command-line-interface/data-book.py @@ -0,0 +1,49 @@ + +from argparse import ArgumentParser +from data_book.api import import_data_book + +import logging + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("Data book API exposed as a command line utility.") + + parser.add_argument("-c", "--max-word-count", default=1e9, help="How many words to add to import to the data book.") + parser.add_argument("-l", "--language", default="dataset", help="The name of the language to import.") + parser.add_argument("-i", "--import", default="fasttext", help="Where to import the data book from.") + parser.add_argument("-o", "--output-path", default="dataset.csv", help="The path to save selected frames.") + parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") + parser.add_argument("--verbose-info", default=False, action="store_true", help="Print out info messages.") + + arguments = vars(parser.parse_args()) + + setup_logging(arguments) + + import_data_book(arguments) + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + +if __name__ == "__main__": + main() + + + diff --git a/platform/micro-services/data-book/source/data_book/importer/fast_text_importer.py b/platform/micro-services/data-book/source/data_book/importer/fast_text_importer.py new file mode 100644 index 00000000..553c5c2f --- /dev/null +++ b/platform/micro-services/data-book/source/data_book/importer/fast_text_importer.py @@ -0,0 +1,205 @@ + +import cloud_hal.storage + +from iso639 import languages + +from urllib.request import urlopen +from smart_open import open +from gtts import gTTS + +import os +import sys +import gzip +import shutil +import csv +import logging + +logger = logging.getLogger(__name__) + +logging.getLogger("gtts.tts").setLevel(logging.CRITICAL) +logging.getLogger("gtts.lang").setLevel(logging.CRITICAL) + +class FastTextImporter: + def __init__(self, config): + self.config = config + + self.storage_client = cloud_hal.storage.client(config=config) + + def import_data_book(self): + # download from FastText website to cache if needed + embeddings_path = self.download_from_fast_text_to_cache() + + # load words + words = self.load_words(embeddings_path) + + # generate audio from Google translate to cache if needed + words_and_audio = self.add_audio_for_words(words) + + # write data book pages + self.write_data_book_pages(words_and_audio) + + def download_from_fast_text_to_cache(self): + language_code = self.get_language_code() + + cache_path = self.get_fasttext_cache_path(language_code) + + logger.debug("Checking for language " + language_code + " at " + cache_path) + + if not self.storage_client.exists(cache_path): + logger.debug("It does not exist, downloading it to cache...") + download_model(language_code, cache_path) + + return cache_path + + def get_language_code(self): + language = languages.get(name=self.config["language"].capitalize()) + + return language.alpha2 + + def get_fasttext_cache_path(self, language_code): + return os.path.join(self.storage_client.get_root(), "cache", "data_book", "fasttext", get_file_name(language_code)) + + def load_words(self, embeddings_path): + embeddings = load_vectors(embeddings_path, int(self.config["max_word_count"])) + + return embeddings.keys() + + def add_audio_for_words(self, words): + audio_for_words = [] + + for word in words: + cache_path = self.get_word_cache_path(word) + + try: + if not self.storage_client.exists(cache_path): + local_path = self.get_local_path_for_word(word) + + tts = gTTS(word, lang=self.get_language_code()) + + os.makedirs(os.path.dirname(local_path), exist_ok=True) + + tts.save(local_path) + + self.storage_client.copy(local_path, cache_path) + + logger.error("Generated audio for word: " + word) + + audio_for_words.append((word, cache_path)) + + except Exception as e: + logger.error("Failed to generate audio for word: " + word + " : " + str(e)) + + return audio_for_words + + def get_word_cache_path(self, word): + return os.path.join(self.storage_client.get_root(), "cache", "data_book", "fasttext", self.get_language_code(), word + ".mp3") + + def get_local_path_for_word(self, word): + return os.path.join("/tmp", "cache", "data_book", "fasttext", self.get_language_code(), "local", word + ".mp3") + + def write_data_book_pages(self, words_and_audio): + with open(self.config["output_path"], "w", newline="") as output_file: + writer = csv.writer(output_file, delimiter=',', quotechar='"') + for item in words_and_audio: + writer.writerow(item) + +def load_vectors(fname, max_word_count): + fin = open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore') + n, d = map(int, fin.readline().split()) + data = {} + for line in fin: + tokens = line.rstrip().split(' ') + data[tokens[0]] = map(float, tokens[1:]) + if len(data) >= max_word_count: + break + + logger.debug("Loaded " + str(len(data)) + " word embeddings.") + + return data + +def _print_progress(downloaded_bytes, total_size): + percent = float(downloaded_bytes) / total_size + bar_size = 50 + bar = int(percent * bar_size) + percent = round(percent * 100, 2) + sys.stdout.write(" (%0.2f%%) [" % percent) + sys.stdout.write("=" * bar) + sys.stdout.write(">") + sys.stdout.write(" " * (bar_size - bar)) + sys.stdout.write("]\r") + sys.stdout.flush() + + if downloaded_bytes >= total_size: + sys.stdout.write('\n') + + +def _download_file(url, write_file_name, chunk_size=2**13): + print("Downloading %s" % url) + response = urlopen(url) + if hasattr(response, 'getheader'): + file_size = int(response.getheader('Content-Length').strip()) + else: + file_size = int(response.info().getheader('Content-Length').strip()) + downloaded = 0 + download_file_name = write_file_name + ".part" + + os.makedirs(os.path.dirname(download_file_name), exist_ok=True) + + with open(download_file_name, 'wb') as f: + while True: + chunk = response.read(chunk_size) + downloaded += len(chunk) + if not chunk: + break + f.write(chunk) + _print_progress(downloaded, file_size) + + os.rename(download_file_name, write_file_name) + +def _download_gz_model(gz_file_name, cache_file_name): + + url = "https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/%s" % gz_file_name + _download_file(url, cache_file_name) + + return True + + +valid_lang_ids = {"af", "sq", "als", "am", "ar", "an", "hy", "as", "ast", + "az", "ba", "eu", "bar", "be", "bn", "bh", "bpy", "bs", + "br", "bg", "my", "ca", "ceb", "bcl", "ce", "zh", "cv", + "co", "hr", "cs", "da", "dv", "nl", "pa", "arz", "eml", + "en", "myv", "eo", "et", "hif", "fi", "fr", "gl", "ka", + "de", "gom", "el", "gu", "ht", "he", "mrj", "hi", "hu", + "is", "io", "ilo", "id", "ia", "ga", "it", "ja", "jv", + "kn", "pam", "kk", "km", "ky", "ko", "ku", "ckb", "la", + "lv", "li", "lt", "lmo", "nds", "lb", "mk", "mai", "mg", + "ms", "ml", "mt", "gv", "mr", "mzn", "mhr", "min", "xmf", + "mwl", "mn", "nah", "nap", "ne", "new", "frr", "nso", + "no", "nn", "oc", "or", "os", "pfl", "ps", "fa", "pms", + "pl", "pt", "qu", "ro", "rm", "ru", "sah", "sa", "sc", + "sco", "gd", "sr", "sh", "scn", "sd", "si", "sk", "sl", + "so", "azb", "es", "su", "sw", "sv", "tl", "tg", "ta", + "tt", "te", "th", "bo", "tr", "tk", "uk", "hsb", "ur", + "ug", "uz", "vec", "vi", "vo", "wa", "war", "cy", "vls", + "fy", "pnb", "yi", "yo", "diq", "zea"} + +def download_model(lang_id, cache_file_name): + """ + Download pre-trained common-crawl vectors from fastText's website + https://fasttext.cc/docs/en/crawl-vectors.html + """ + if lang_id not in valid_lang_ids: + raise Exception("Invalid lang id. Please select among %s" % + repr(valid_lang_ids)) + + _download_gz_model(get_file_name(lang_id), cache_file_name) + + return cache_file_name + +def get_file_name(lang_id): + file_name = "cc.%s.300.vec" % lang_id + gz_file_name = "%s.gz" % file_name + + return gz_file_name + + diff --git a/platform/micro-services/data-book/source/data_book/importer/importer_factory.py b/platform/micro-services/data-book/source/data_book/importer/importer_factory.py new file mode 100644 index 00000000..4ed4e937 --- /dev/null +++ b/platform/micro-services/data-book/source/data_book/importer/importer_factory.py @@ -0,0 +1,16 @@ + +from data_book.importer.fast_text_importer import FastTextImporter + +class ImporterFactory: + def __init__(self, config): + self.config = config + + def create(self): + + assert self.config["import"] == "fasttext" + + return FastTextImporter(self.config) + + + + diff --git a/platform/micro-services/data-export/README.md b/platform/micro-services/data-export/README.md new file mode 100644 index 00000000..85a79ca3 --- /dev/null +++ b/platform/micro-services/data-export/README.md @@ -0,0 +1,13 @@ + +# Data Export + +Exports datasets as tar archives. + +# Usage + +peoples-speech data-export dataset dataset.tar.gz + +# Perf Requirements + +100k hours of data exported in 1 day + diff --git a/platform/micro-services/data-export/source/add-clean-text.py b/platform/micro-services/data-export/source/add-clean-text.py new file mode 100644 index 00000000..d61c099f --- /dev/null +++ b/platform/micro-services/data-export/source/add-clean-text.py @@ -0,0 +1,86 @@ +from argparse import ArgumentParser +import logging +import csv +import os +import json +from smart_open import open +from cleantext import clean + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("Clean text.") + + parser.add_argument("-i", "--input-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.6/test.csv", + help = "The output path to load the dataset from.") + parser.add_argument("-o", "--output-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.6/test-cleaned-transcripts.csv", + help = "The output path tosave cleaned transcripts.") + parser.add_argument("--to-lower", default = False, action="store_true", + help = "Lower case.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + clean_transcripts(arguments) + +def clean_transcripts(arguments): + samples = load_csv(arguments["input_path"]) + + updated_samples = update_samples(samples, arguments) + + with open(arguments["output_path"], "w", newline="") as output_csv_file: + csv_writer = csv.writer(output_csv_file, delimiter=',', quotechar='"') + for sample in updated_samples: + csv_writer.writerow(sample) + +def load_csv(csv_path): + new_samples = [] + with open(csv_path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + + for row in reader: + path, caption = row[0], row[1] + + metadata = {} + if len(row) >= 3: + if len(row[2]) > 0: + metadata = json.loads(row[2]) + + yield {"path" : path, "caption" : caption, "metadata" : metadata} + +def update_samples(samples, arguments): + for sample in samples: + cleaned_caption = clean(sample["caption"], lower=arguments["to_lower"], no_line_breaks=True).strip() + metadata = sample["metadata"] + logger.debug("For " + sample["path"]) + metadata["pre-cleaned-transcript"] = sample["caption"] + logger.debug("Cleaned transcript from '" + sample["caption"] + "' to '" + cleaned_caption + "'") + yield (sample["path"], cleaned_caption, json.dumps(metadata)) + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +if __name__ == "__main__": + main() + + + diff --git a/platform/micro-services/data-export/source/add-duration-metadata.py b/platform/micro-services/data-export/source/add-duration-metadata.py new file mode 100644 index 00000000..5afdf09a --- /dev/null +++ b/platform/micro-services/data-export/source/add-duration-metadata.py @@ -0,0 +1,233 @@ +import concurrent.futures +from argparse import ArgumentParser +import logging +import csv +import os +import json +from google.cloud import storage +from smart_open import open +import audiofile + +logger = logging.getLogger(__name__) +storage_client = storage.Client() + +def main(): + parser = ArgumentParser("Add duration metadata to data.") + + parser.add_argument("-i", "--input-path", + default = "gs://the-peoples-speech-aws-import/librispeech-formatted/train-clean-100-clean-metadata.csv", + help = "The output path to load the dataset from.") + parser.add_argument("-o", "--output-path", + default = "gs://the-peoples-speech-aws-import/librispeech-formatted/train-clean-100-clean-metadata-durations.csv", + help = "The output path to save dataset with metadata.") + parser.add_argument("--worker-count", default = 8, + help = "The number of worker threads.") + parser.add_argument("--cache-directory", default = "data", + help = "The local path to cache.") + parser.add_argument("-b", "--batch-size", default = 256, + help = "The number of audio files to process at one time.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + arguments["system"] = {"cache-directory" : arguments["cache_directory"]} + + add_duration_metadata(arguments) + +def add_duration_metadata(arguments): + samples = load_csv(arguments["input_path"]) + + with open(arguments["output_path"], "w", newline="") as output_csv_file: + csv_writer = csv.writer(output_csv_file, delimiter=',', quotechar='"') + converter = AudioConverter(arguments, samples, csv_writer) + + converter.run() + +def load_csv(csv_path): + new_samples = [] + with open(csv_path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + + for row in reader: + path, caption = row[0], row[1] + + metadata = {} + if len(row) >= 3: + if len(row[2]) > 0: + metadata = json.loads(row[2]) + + yield {"path" : path, "caption" : caption, "metadata" : metadata} + +class AudioConverter: + def __init__(self, arguments, samples, csv_writer): + self.arguments = arguments + self.samples = samples + self.csv_writer = csv_writer + + def run(self): + total_duration = 0.0 + total_bytes = 0.0 + with concurrent.futures.ThreadPoolExecutor(max_workers=int(self.arguments["worker_count"])) as executor: + while True: + sample_batch = self.get_next_batch() + + if len(sample_batch) == 0: + break + + future_to_data = {executor.submit(add_duration, self.arguments, path, metadata): (path, transcript, metadata) for + path, transcript, metadata in sample_batch} + + for future in concurrent.futures.as_completed(future_to_data): + path, transcript, metadata = future_to_data[future] + try: + new_metadata = future.result() + self.csv_writer.writerow([path, transcript, json.dumps(new_metadata)]) + + duration = new_metadata["duration_seconds"] + total_duration += duration + + byte_count = new_metadata["size_in_bytes"] + total_bytes += new_metadata["size_in_bytes"] + + logger.debug(" duration is %s seconds out of total %s (%s / %s bytes) with audio %s" % + (sizeof_fmt(duration), sizeof_fmt(total_duration / 3600.0, suffix='h'), + sizeof_fmt(byte_count, suffix='B'), + sizeof_fmt(total_bytes, suffix='B'), path)) + except Exception as exc: + print('%r generated an exception: %s' % (path, exc)) + + def get_next_batch(self): + batch = [] + + for i in range(int(self.arguments["batch_size"])): + try: + row = next(self.samples) + + path = row["path"] + transcript = row["caption"] + metadata = row["metadata"] + + batch.append((path, transcript, metadata)) + except StopIteration: + break + + + return batch + +def add_duration(arguments, path, metadata): + local_path = LocalFileCache(arguments, path).get_path() + audio_duration = audiofile.duration(local_path) + metadata["duration_seconds"] = audio_duration + metadata["size_in_bytes"] = os.path.getsize(local_path) + + os.remove(local_path) + return metadata + +def sizeof_fmt(num, suffix='s'): + num = float(num) + for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, 'Yi', suffix) + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +class LocalFileCache: + """ Supports caching. Currently it supports read-only access to GCS. + """ + + def __init__(self, config, remote_path, create=True): + self.config = config + self.remote_path = remote_path + + self.local_path = self.compute_local_path() + + if create: + self.download_if_remote() + + def get_path(self): + return self.local_path + + def download_if_remote(self): + if not self.is_remote_path(self.remote_path): + return + + self.download() + + def download(self): + if os.path.exists(self.get_path()): + logger.info(" using cached file '" + self.get_path() + "'") + return + + directory = os.path.dirname(self.get_path()) + + os.makedirs(directory, exist_ok=True) + + bucket, key = get_bucket_and_prefix(self.remote_path) + + logger.info( + "Downloading '" + self.remote_path + "' to '" + self.get_path() + "'" + ) + + bucket = storage_client.get_bucket(bucket) + blob = bucket.get_blob(key) + + blob.download_to_filename(self.local_path) + + def is_remote_path(self, path): + return path.find("gs:") == 0 + + def compute_local_path(self): + if not self.is_remote_path(self.remote_path): + return self.remote_path + bucket, key = get_bucket_and_prefix(self.remote_path) + return os.path.join(self.config["system"]["cache-directory"], key) + +def get_bucket_and_prefix(path): + parts = split_all(path[5:]) + + return parts[0], os.path.join(*parts[1:]) + +def split_all(path): + allparts = [] + while True: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts + +if __name__ == "__main__": + main() + + + + + diff --git a/platform/micro-services/data-export/source/add-google-speech-transcript.py b/platform/micro-services/data-export/source/add-google-speech-transcript.py new file mode 100644 index 00000000..bf8458ac --- /dev/null +++ b/platform/micro-services/data-export/source/add-google-speech-transcript.py @@ -0,0 +1,120 @@ + +from argparse import ArgumentParser +import logging +import csv +import os +import json +import tarfile +import io +from smart_open import open +from google.cloud import speech_v1 as speech + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser(".") + + parser.add_argument("-i", "--input-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.6/test.csv", + help = "The output path to load the dataset from.") + parser.add_argument("-o", "--output-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.6/test-with-transcripts.csv", + help = "The output path to save new transcripts.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + add_transcripts(arguments) + +def add_transcripts(arguments): + samples = load_csv(arguments["input_path"]) + + updated_samples = update_samples(samples) + + with open(arguments["output_path"], "w", newline="") as output_csv_file: + csv_writer = csv.writer(output_csv_file, delimiter=',', quotechar='"') + for sample in updated_samples: + csv_writer.writerow(sample) + +def load_csv(csv_path): + new_samples = [] + with open(csv_path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + + for row in reader: + path, caption = row[0], row[1] + + metadata = {} + if len(row) >= 3: + if len(row[2]) > 0: + metadata = json.loads(row[2]) + + yield {"path" : path, "caption" : caption, "metadata" : metadata} + +def update_samples(samples): + for sample in samples: + metadata = sample["metadata"] + logger.debug("For " + sample["path"]) + metadata["google-speech-api-transcript"] = get_transcript(sample["path"]) + logger.debug("Transcribed '" + sample["caption"] + "' to '" + metadata["google-speech-api-transcript"] + "'") + yield (sample["path"], sample["caption"], json.dumps(metadata)) + +speech_client = speech.SpeechClient() + +def get_transcript(path): + + # The language of the supplied audio + language_code = "en-US" + + # Sample rate in Hertz of the audio data sent + sample_rate_hertz = 16000 + + # Encoding of audio data sent. This sample sets this explicitly. + # This field is optional for FLAC and WAV audio formats. + #encoding = speech.RecognitionConfig.AudioEncoding.MP3 + config = { + "language_code": language_code, + #"enable_separate_recognition_per_channel": True, + #"audio_channel_count": 2, + "sample_rate_hertz": sample_rate_hertz, + # "encoding": encoding, + } + audio = {"uri": path} + + response = speech_client.recognize(config=config, audio=audio) + + transcript = "" + + for result in response.results: + # First alternative is the most probable result + alternative = result.alternatives[0] + transcript += alternative.transcript + + return transcript + + + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +if __name__ == "__main__": + main() + + diff --git a/platform/micro-services/data-export/source/add-librispeech-metadata.py b/platform/micro-services/data-export/source/add-librispeech-metadata.py new file mode 100644 index 00000000..6fb07301 --- /dev/null +++ b/platform/micro-services/data-export/source/add-librispeech-metadata.py @@ -0,0 +1,121 @@ +from argparse import ArgumentParser +import logging +import csv +import os +import json +from smart_open import open + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("Add metadata to librispeech data.") + + parser.add_argument("-i", "--input-path", default = "gs://the-peoples-speech-aws-import/librispeech-formatted/train-clean-100.csv", + help = "The output path to load the dataset from.") + parser.add_argument("-m", "--metadata-path", default = "gs://the-peoples-speech-aws-import/librispeech-formatted/SPEAKERS.txt", + help = "The path to load the metadata from.") + parser.add_argument("-o", "--output-path", default = "gs://the-peoples-speech-aws-import/librispeech-formatted/train-clean-100-metadata.csv", + help = "The output path to save dataset with metadata.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + add_metadata(arguments) + +def add_metadata(arguments): + samples = load_csv(arguments["input_path"]) + metadata = load_metadata(arguments["metadata_path"]) + + updated_samples = update_samples(samples, metadata) + + with open(arguments["output_path"], "w", newline="") as output_csv_file: + csv_writer = csv.writer(output_csv_file, delimiter=',', quotechar='"') + for sample in updated_samples: + csv_writer.writerow(sample) + +def decomment(csvfile): + for row in csvfile: + raw = row.split(';')[0].strip() + if raw: yield raw + +def load_metadata(speakers_path): + metadata = {} + with open(speakers_path) as speakers_file: + csv_reader = csv.reader(decomment(speakers_file), delimiter='|', quotechar='"') + + #14 | F | train-clean-360 | 25.03 | Kristin LeMoine + #60 | M | train-clean-100 | 20.18 | |CBW|Simon + for row in csv_reader: + if len(row) > 5: + row[4] = "|".join(row[4:]) + + speaker_id = row[0].strip() + gender = row[1].strip() + dataset_name = row[2].strip() + hours = row[3].strip() + name = row[4].strip() + + metadata[speaker_id] = {"data_source" : "librispeech", + "speaker_id" : speaker_id, "gender" : gender, + "librispeech_split_name" : dataset_name, + "hours_per_speaker" : hours, "speaker_name" : name} + + logger.debug("Found metadata for speaker id: '" + str(speaker_id) + "': " + str(metadata[speaker_id])) + + return metadata + +def load_csv(csv_path): + new_samples = [] + with open(csv_path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + + for row in reader: + path, caption = row[0], row[1] + + metadata = {} + if len(row) >= 3: + if len(row[2]) > 0: + metadata = json.loads(row[2]) + + yield {"path" : path, "caption" : caption, "metadata" : metadata} + +def update_samples(samples, metadata): + for sample in samples: + name = get_speaker_id_for_sample(sample["path"]) + logger.debug("For " + sample["path"]) + sample_metadata = metadata[name] + logger.debug("Added metadata " + str(sample_metadata)) + yield (sample["path"], sample["caption"], json.dumps(sample_metadata)) + +def get_speaker_id_for_sample(path): + basename = os.path.basename(path) + return basename.split("-")[0] + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +if __name__ == "__main__": + main() + + + + diff --git a/platform/micro-services/data-export/source/convert-audio-format.py b/platform/micro-services/data-export/source/convert-audio-format.py new file mode 100644 index 00000000..c8de99b4 --- /dev/null +++ b/platform/micro-services/data-export/source/convert-audio-format.py @@ -0,0 +1,278 @@ +from argparse import ArgumentParser +import logging +import csv +import json +import os +from google.cloud import storage +from smart_open import open +import concurrent.futures + +from pydub import AudioSegment + +logger = logging.getLogger(__name__) + +storage_client = storage.Client() + +def main(): + parser = ArgumentParser("Convert all of the audio files in a csv to the specified format, update the csv to point at the new files.") + + parser.add_argument("-i", "--input-path", default = "gs://the-peoples-speech-aws-import/common-voice/train.csv", + help = "The path to the dataset stored in csv format.") + parser.add_argument("--max-count", default = 1e9, + help = "The maximum number of audio samples to extract.") + parser.add_argument("--cache-directory", default = "data", + help = "The local path to cache.") + parser.add_argument("-o", "--output-path", default = "gs://the-peoples-speech-aws-import/common-voice/train-flac.csv", + help = "The output path to save the dataset.") + parser.add_argument("--worker-count", default = 8, + help = "The number of worker threads.") + parser.add_argument("--sampling-rate", default = 16000, + help = "The sampling rate for the audio.") + parser.add_argument("-f", "--format", default = "flac", + help = "The audio format to convert to.") + parser.add_argument("-b", "--batch-size", default = 256, + help = "The number of audio files to process at one time.") + parser.add_argument("-v", "--verbose", default = False, + action="store_true", + help = "Set the log level to debug, " + "printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + arguments["system"] = {"cache-directory" : arguments["cache_directory"]} + + setup_logger(arguments) + + convert_audio_format(arguments) + + +def convert_audio_format(arguments): + with open(arguments["output_path"], "w", newline="") as output_csv_file, \ + open(arguments["input_path"]) as input_csv_file: + csv_reader = csv.reader(input_csv_file, delimiter=',', quotechar='"') + csv_writer = csv.writer(output_csv_file, delimiter=',', quotechar='"') + + converter = AudioConverter(arguments, csv_reader, csv_writer) + + converter.run() + +class AudioConverter: + def __init__(self, arguments, csv_reader, csv_writer): + self.arguments = arguments + self.csv_reader = csv_reader + self.csv_writer = csv_writer + + def run(self): + total_bytes = 0 + with concurrent.futures.ThreadPoolExecutor(max_workers=int(self.arguments["worker_count"])) as executor: + while True: + sample_batch = self.get_next_batch() + + if len(sample_batch) == 0: + break + + future_to_data = {executor.submit(convert_file, self.arguments, path, updated_path): (updated_path, path, transcript, metadata) for + updated_path, path, transcript, metadata in sample_batch} + + for future in concurrent.futures.as_completed(future_to_data): + updated_path, path, transcript, metadata = future_to_data[future] + try: + byte_count, ratio = future.result() + total_bytes += byte_count + self.csv_writer.writerow([updated_path, transcript, metadata]) + + logger.debug("converted %s (%sx) / %s bytes from %s " % (sizeof_fmt(byte_count), ratio, sizeof_fmt(total_bytes), path)) + except Exception as exc: + print('%r generated an exception: %s' % (path, exc)) + + def get_next_batch(self): + batch = [] + + for i in range(int(self.arguments["batch_size"])): + try: + row = next(self.csv_reader) + + path = row[0] + transcript = row[1] + metadata = "" + + if len(row) > 2: + metadata = row[2] + + updated_path = update_path(path, self.arguments["format"]) + + batch.append((updated_path, path, transcript, metadata)) + except StopIteration: + break + + + return batch + +def sizeof_fmt(num, suffix='B'): + num = float(num) + for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, 'Yi', suffix) + +def convert_file(config, path, updated_path): + if blob_exists(updated_path): + logger.debug("Skipping existing file '" + updated_path + "'") + return 1, 1 + + local_path = LocalFileCache(config, path).get_path() + updated_local_path = update_path(local_path, get_format(updated_path)) + + logger.debug("Converting from " + local_path + " to " + updated_local_path + " ( " + updated_path + " )") + + audio = AudioSegment.from_file(local_path, os.path.splitext(local_path)[1][1:]) + audio.set_frame_rate(int(config["sampling_rate"])) + + audio.export(updated_local_path, format=get_format(updated_path), parameters=["-compression_level", "4", "-ac", "1"]) + + # upload the file + bucket_name, key = get_bucket_and_prefix(updated_path) + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(key) + blob.upload_from_filename(local_path) + + original_size = os.path.getsize(local_path) + updated_size = os.path.getsize(updated_local_path) + + os.remove(local_path) + os.remove(updated_local_path) + + return updated_size, updated_size / original_size + +def blob_exists(path): + bucket_name, prefix = get_bucket_and_prefix(path) + try: + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(prefix) + return blob.exists() + except Exception as e: + logger.debug(path + " does not exist: exception: " + str(e)) + return False + + return True + +def get_format(path): + pre, ext = os.path.splitext(path) + + return ext[1:] + +def update_path(path, format_name): + pre, ext = os.path.splitext(path) + + return pre + "." + format_name + +def get_bucket_and_prefix(path): + parts = split_all(path[5:]) + + return parts[0], os.path.join(*parts[1:]) + +def split_all(path): + allparts = [] + while True: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s -' + ' %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +class LocalFileCache: + """ Supports caching. Currently it supports read-only access to GCS. + """ + + def __init__(self, config, remote_path, create=True): + self.config = config + self.remote_path = remote_path + + self.local_path = self.compute_local_path() + + if create: + self.download_if_remote() + + def get_path(self): + return self.local_path + + def download_if_remote(self): + if not self.is_remote_path(self.remote_path): + return + + self.download() + + def download(self): + if os.path.exists(self.get_path()): + logger.info(" using cached file '" + self.get_path() + "'") + return + + directory = os.path.dirname(self.get_path()) + + os.makedirs(directory, exist_ok=True) + + bucket, key = get_bucket_and_prefix(self.remote_path) + + logger.info( + "Downloading '" + self.remote_path + "' to '" + self.get_path() + "'" + ) + + bucket = storage_client.get_bucket(bucket) + blob = bucket.get_blob(key) + + blob.download_to_filename(self.local_path) + + def is_remote_path(self, path): + return path.find("gs:") == 0 + + def compute_local_path(self): + if not self.is_remote_path(self.remote_path): + return self.remote_path + bucket, key = get_bucket_and_prefix(self.remote_path) + return os.path.join(self.config["system"]["cache-directory"], key) + + + + +main() + + + + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/convert-cc-search.py b/platform/micro-services/data-export/source/convert-cc-search.py new file mode 100644 index 00000000..9dfd87cc --- /dev/null +++ b/platform/micro-services/data-export/source/convert-cc-search.py @@ -0,0 +1,399 @@ +from argparse import ArgumentParser +import logging +import csv +import json +import os +import queue +import hashlib +import threading +import gc + +from google.cloud import storage + +from pydub import AudioSegment + +logger = logging.getLogger(__name__) + +from smart_open import open + +storage_client = storage.Client() + +def main(): + parser = ArgumentParser("This program converts the DSAlign " + "data format to the default CSV audio file format.") + + parser.add_argument("-i", "--input-path", default = "gs://the-peoples-speech-west-europe/archive_org/Aug_18_2020", + help = "The GCP path to the audio dataset.") + parser.add_argument("--aligned-path", default = "gs://the-peoples-speech-west-europe/archive_org/Aug_18_2020_aligned_data_9_15_20", + help = "The GCP path to the alignment dataset.") + parser.add_argument("--max-count", default = 1e9, + help = "The maximum number of audio samples to extract.") + parser.add_argument("--cache-directory", default = "", + help = "The local path to cache.") + parser.add_argument("-o", "--output-path", default = "gs://the-peoples-speech-west-europe/archive_org/v0.2", + help = "The output path to save the dataset.") + parser.add_argument("--worker-count", default = 4, + help = "The number of worker threads.") + parser.add_argument("-v", "--verbose", default = False, + action="store_true", + help = "Set the log level to debug, " + "printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + arguments["system"] = {"cache-directory" : arguments["cache_directory"]} + + setup_logger(arguments) + + convert_cc_search_to_csv(arguments) + +def convert_cc_search_to_csv(arguments): + + directory = arguments["output_path"] + + logger.debug("Checking directory: " + directory) + if not os.path.exists(directory): + logger.debug("Making directory: " + directory) + os.makedirs(directory) + + with open(os.path.join(arguments["output_path"], "data.csv"), "w", newline="") as output_csv_file: + csv_writer = csv.writer(output_csv_file, delimiter=',', quotechar='"') + update_csv(arguments, csv_writer) + +def update_csv(arguments, csv_writer): + mp3_files = dict(get_mp3_files(arguments["input_path"]), **get_mp3_files(arguments["output_path"])) + + total_count = 0 + + file_uploader = FileUploader(arguments) + + for bucket_name, file_name in get_all_object_paths(arguments): + if not is_aligned_file(file_name): + continue + + alignment_file_name = "gs://" + os.path.join(bucket_name, file_name) + alignments, mp3_path = load_alignments(arguments, alignment_file_name) + + if all_alignments_exist(arguments, mp3_path, alignments, mp3_files): + logger.debug("Skipping mp3 from " + mp3_path + " which is already fully converted.") + continue + + if not blob_exists(mp3_files, mp3_path): + continue + + mp3_size = get_blob_size(mp3_path) + + if mp3_size > 250e6: + logger.debug("Skipping mp3 from " + mp3_path + " with " + str(mp3_size / 1e6) + "MB which is too big") + continue + + logger.debug("Loading mp3 from " + mp3_path + " with " + str(mp3_size / 1e6) + "MB") + mp3 = load_audio(mp3_path, arguments) + + logger.debug("Extracting alignments from " + str(alignment_file_name) + ", " + str(mp3_path)) + + for entry in alignments: + start = entry["start"] + end = entry["end"] + + text = entry["aligned"] + + save_training_sample(mp3_files, file_uploader, csv_writer, mp3, mp3_path, start, end, text, entry, arguments, total_count) + + total_count += 1 + + if total_count >= int(arguments["max_count"]): + break + + del alignments + + delete_audio(mp3, mp3_path, alignment_file_name, arguments) + + if total_count >= int(arguments["max_count"]): + break + + file_uploader.join() + +def all_alignments_exist(arguments, mp3_path, alignments, mp3_files): + for entry in alignments: + start = entry["start"] + end = entry["end"] + + path = get_output_path(arguments, mp3_path, start, end) + + if not blob_exists(mp3_files, path): + return False + + return True + +def get_mp3_files(audio_path): + logger.debug("Getting MP3 files under " + audio_path) + + # Note: Client.list_blobs requires at least package version 1.17.0. + bucket_name, prefix = get_bucket_and_prefix(audio_path) + blobs = storage_client.list_blobs(bucket_name, prefix=prefix) + + mp3_files = {} + + for blob in blobs: + if is_mp3(blob.name): + path = os.path.join("gs://" + bucket_name, blob.name) + mp3_files[path] = get_key(blob.name) + + logger.debug(" Found " + str(len(mp3_files)) + " mp3 files") + + return mp3_files + +def get_key(path): + parts = split_all(path) + return os.path.splitext(parts[-2] + "-" + parts[-1])[0] + +def is_mp3(path): + return os.path.splitext(path)[1] == ".mp3" or os.path.splitext(path)[1] == ".wav" + +def blob_exists(paths, path): + return path in paths + +def get_blob_size(path): + bucket_name, prefix = get_bucket_and_prefix(path) + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.get_blob(prefix) + return blob.size + +def is_aligned_file(path): + return path.find("aligned.json") != -1 + +def load_alignments(arguments, path): + logger.debug("Loading " + path) + local_cache = LocalFileCache(arguments, path).get_path() + with open(local_cache) as json_file: + return json.load(json_file), get_mp3_path_for_aligned_file(path) + +def get_mp3_path_for_aligned_file(path): + # gs://the-peoples-speech-west-europe/archive_org/Aug_18_2020_aligned_data_9_15_20/CAPTIONED_DATA/output/10_10_2017_Essex_Junction_Trustees/aligned.json + # gs://the-peoples-speech-west-europe/archive_org/Aug_18_2020/CAPTIONED_DATA/10_10_2017_Essex_Junction_Trustees/10_10_2017_Essex_Junction_Trustees.mp3 + + parts = split_all(path[5:]) + + return "gs://" + os.path.join(*(parts[:2] + ["Aug_18_2020", "CAPTIONED_DATA"] + parts[-2:-1] + parts[-2:-1])) + ".mp3" + +def load_audio(path, arguments): + return MP3File(path, arguments) + +class MP3File: + def __init__(self, path, arguments): + self.path = path + self.arguments = arguments + self.mp3 = None + + def get(self): + if self.mp3 is None: + local_cache = LocalFileCache(self.arguments, self.path).get_path() + + self.mp3 = AudioSegment.from_mp3(local_cache) + + return self.mp3 + +def delete_audio(mp3, mp3_path, alignment_file_name, arguments): + del mp3 + gc.collect() + local_cache = LocalFileCache(arguments, alignment_file_name, create=False).get_path() + logger.debug("Deleting cache file " + local_cache) + if os.path.exists(local_cache): + os.remove(local_cache) + + local_mp3_cache = LocalFileCache(arguments, mp3_path, create=False).get_path() + logger.debug("Deleting mp3 cache file " + local_mp3_cache) + if os.path.exists(local_mp3_cache): + os.remove(local_mp3_cache) + +def extract_audio(audio, start, end): + return audio.get()[start:end] + +def save_training_sample(mp3s, file_uploader, csv_writer, audio, input_path, start, end, text, entry, arguments, total_count): + path = get_output_path(arguments, input_path, start, end) + + if not blob_exists(mp3s, path): + local_path = get_local_path(arguments, input_path, start, end) + + directory = os.path.dirname(local_path) + + if not os.path.exists(directory): + os.makedirs(directory) + + logger.debug("Saving sample: " + path + " at local path " + local_path) + + audio_segment = extract_audio(audio, start, end) + + audio_segment.export(local_path, format="wav") + + file_uploader.upload(path, local_path) + + csv_writer.writerow([path, text, json.dumps(entry)]) + +def get_output_path(arguments, input_path, start, end): + return os.path.join(arguments["output_path"], "data", "audio-" + hash_function(input_path + "-" + str(start) + "-" + str(end)) + ".wav") + +def hash_function(data): + return hashlib.sha256(data.encode('utf-8')).hexdigest() + +def get_local_path(arguments, input_path, start, end): + bucket, key = get_bucket_and_prefix(get_output_path(arguments, input_path, start, end)) + return os.path.join(arguments["system"]["cache-directory"], key) + +def get_all_object_paths(arguments): + + bucket_name, prefix = get_bucket_and_prefix(arguments["aligned_path"]) + blobs = storage_client.list_blobs(bucket_name, prefix=prefix) + + for blob in blobs: + yield bucket_name, blob.name + +def get_bucket_and_prefix(path): + parts = split_all(path[5:]) + + return parts[0], os.path.join(*parts[1:]) + +def split_all(path): + allparts = [] + while True: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s -' + ' %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +class FileUploader: + def __init__(self, arguments): + self.queue = queue.Queue(maxsize=512) + self.threads = [] + + for i in range(int(arguments["worker_count"])): + thread = threading.Thread(target=upload_files_worker, args=(self.queue,)) + thread.start() + self.threads.append(thread) + + def upload(self, path, local_path): + self.queue.put((path, local_path, False)) + + def join(self): + + for thread in self.threads: + self.queue.put((None, None, True)) + + for thread in self.threads: + thread.join() + +def upload_files_worker(queue): + while True: + path, local_path, is_finished = queue.get() + + if is_finished: + break + + logger.debug("Uploading " + local_path + " to " + path) + + try: + bucket_name, key = get_bucket_and_prefix(path) + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(key) + blob.upload_from_filename(local_path) + except: + pass + + os.remove(local_path) + +class LocalFileCache: + """ Supports caching. Currently it supports read-only access to GCS. + """ + + def __init__(self, config, remote_path, create=True): + self.config = config + self.remote_path = remote_path + + self.local_path = self.compute_local_path() + + if create: + self.download_if_remote() + + def get_path(self): + return self.local_path + + def download_if_remote(self): + if not self.is_remote_path(self.remote_path): + return + + self.download() + + def download(self): + if os.path.exists(self.get_path()): + logger.info(" using cached file '" + self.get_path() + "'") + return + + directory = os.path.dirname(self.get_path()) + + os.makedirs(directory, exist_ok=True) + + bucket, key = get_bucket_and_prefix(self.remote_path) + + logger.info( + "Downloading '" + self.remote_path + "' to '" + self.get_path() + "'" + ) + + bucket = storage_client.get_bucket(bucket) + blob = bucket.get_blob(key) + + blob.download_to_filename(self.local_path) + + def is_remote_path(self, path): + return path.find("gs:") == 0 + + def compute_local_path(self): + if not self.is_remote_path(self.remote_path): + return self.remote_path + bucket, key = get_bucket_and_prefix(self.remote_path) + return os.path.join(self.config["system"]["cache-directory"], key) + + + + +main() + + + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/convert-common-voice.py b/platform/micro-services/data-export/source/convert-common-voice.py new file mode 100644 index 00000000..d89c846b --- /dev/null +++ b/platform/micro-services/data-export/source/convert-common-voice.py @@ -0,0 +1,122 @@ + +from argparse import ArgumentParser +import logging +import csv +import json +import os +import shutil + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("This program converts the COMMON-VOICE data format to " + "the default CSV audio file format.") + + parser.add_argument("-i", "--input-path", default = "", + help = "The path to the COMMON-VOICE dataset.") + parser.add_argument("-o", "--output-path", default = "", + help = "The output path to save the dataset.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + convert_common_voice_to_csv(arguments) + +def convert_common_voice_to_csv(arguments): + all_files = get_all_files(arguments) + + if not os.path.exists(arguments["output_path"]): + os.makedirs(arguments["output_path"]) + os.makedirs(os.path.join(arguments["output_path"], "clips")) + + train_csv_path = os.path.join(arguments["output_path"], "train.csv") + test_csv_path = os.path.join(arguments["output_path"], "test.csv") + dev_csv_path = os.path.join(arguments["output_path"], "dev.csv") + + with open(train_csv_path, "w") as output_train_file, \ + open(test_csv_path, "w") as output_test_file, \ + open(dev_csv_path, "w") as output_dev_file: + train_writer = csv.writer(output_train_file, delimiter=',', quotechar='"') + test_writer = csv.writer(output_test_file, delimiter=',', quotechar='"') + dev_writer = csv.writer(output_test_file, delimiter=',', quotechar='"') + + for index, filename in enumerate(all_files): + with open(filename) as input_file: + reader = csv.reader(input_file, delimiter='\t', quotechar='"') + + next(reader) + + for row in reader: + label = row[2] + path = row[1] + + base = os.path.split(filename)[0] + + utterance_path = os.path.join(base, "clips", path) + + new_filename = os.path.join(arguments["output_path"], "clips", path) + + logger.debug("Copying from " + utterance_path + " to " + new_filename) + shutil.copy(utterance_path, new_filename) + + if is_test(filename): + test_writer.writerow([new_filename, label]) + elif is_dev(filename): + dev_writer.writerow([new_filename, label]) + elif is_train(filename): + train_writer.writerow([new_filename, label]) + +def is_test(filename): + return filename.find("test") != -1 + +def is_dev(filename): + return filename.find("dev") != -1 + +def is_train(filename): + return filename.find("train") != -1 + +def is_tsv(filename): + return os.path.splitext(filename)[1] == ".tsv" + +def get_all_files(arguments): + all_files = [] + + for root, directories, files in os.walk(arguments["input_path"]): + all_files += [os.path.join(root, f) for f in files if is_tsv(f)] + + return sorted(all_files) + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +main() + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/convert-dipco-dataset-format.py b/platform/micro-services/data-export/source/convert-dipco-dataset-format.py new file mode 100644 index 00000000..b7829986 --- /dev/null +++ b/platform/micro-services/data-export/source/convert-dipco-dataset-format.py @@ -0,0 +1,187 @@ + +from argparse import ArgumentParser +import logging +import csv +import json +import os +from datetime import datetime + +from pydub import AudioSegment + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("This program converts the DiPCo data format to " + "the default CSV audio file format.") + + parser.add_argument("-i", "--input-path", default = "", + help = "The path to the DiPCo dataset.") + parser.add_argument("-o", "--output-path", default = "", + help = "The output path to save the dataset.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out " + "detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + convert_dipco_to_csv(arguments) + +def convert_dipco_to_csv(arguments): + all_files = get_all_files(arguments) + + if not os.path.exists(arguments["output_path"]): + os.makedirs(arguments["output_path"]) + + train_csv_path = os.path.join(arguments["output_path"], "train.csv") + test_csv_path = os.path.join(arguments["output_path"], "test.csv") + + with open(train_csv_path, "w") as output_train_file, \ + open(test_csv_path, "w") as output_test_file: + train_writer = csv.writer(output_train_file, delimiter=',', quotechar='"') + test_writer = csv.writer(output_test_file, delimiter=',', quotechar='"') + + for index, filename in enumerate(all_files): + rows = json.load(open(filename)) + + for row in rows: + items = expand_row(row, filename) + + for item in items: + new_filename, label = copy_data(item, arguments) + + if is_test(filename): + test_writer.writerow([new_filename, label]) + else: + train_writer.writerow([new_filename, label]) + +def expand_row(row, filename): + results = [] + + label = row["words"] + + for key, start_time in row["start_time"].items(): + end_time = row["end_time"][key] + + if key == 'close-talk': + results.append((filename, label, row["speaker_id"], start_time, end_time)) + else: + for i in range(1,8): + results.append((filename, label, key + ".CH" + str(i), start_time, end_time)) + + return results + +def copy_data(item, arguments): + filename, label, key, start_time, end_time = item + + source_filename = get_source_filename(filename, key) + + target_filename = get_target_filename(arguments, filename, key, start_time) + + extract_wav(target_filename, source_filename, start_time, end_time) + + return (target_filename, label) + +def get_source_filename(filename, key): + s_id = get_s_id(filename) + modifier = get_modifier(key) + + base = get_base(filename) + + return os.path.join(base, s_id + modifier + ".wav") + +def get_s_id(filename): + return os.path.splitext(os.path.basename(filename))[0] + +def get_modifier(key): + return "_" + key + +def get_base(filename): + directory_switch = "eval" if is_test(filename) else "dev" + + base, tail = os.path.split(filename) + + while tail != 'transcriptions' and base != '': + base, tail = os.path.split(base) + + return os.path.join(base, "audio", directory_switch) + +def get_target_filename(arguments, filename, key, start_time): + directory_switch = "eval" if is_test(filename) else "dev" + base = os.path.join(arguments["output_path"], directory_switch) + + if not os.path.exists(base): + os.makedirs(base) + + s_id = get_s_id(filename) + modifier = get_modifier(key) + timestamp = start_time.replace(":", "_") + return os.path.join(base, s_id + modifier + timestamp + ".wav") + +def extract_wav(target_filename, source_filename, start_time, end_time): + logger.debug("extracting data from " + source_filename + + " [" + start_time + ", " + end_time + "]") + + wav = AudioSegment.from_wav(source_filename) + + start_milliseconds = get_milliseconds(start_time) + end_milliseconds = get_milliseconds( end_time) + + wav_slice = wav[start_milliseconds:end_milliseconds] + + wav_slice.export(target_filename, format='wav') + +def get_milliseconds(time): + split_time = time.split(":") + + seconds = float(split_time[0]) * 3600 + float(split_time[1]) * 60 + float(split_time[2]) + + return seconds * 1000 + +def is_test(filename): + return filename.find("eval") != -1 + +def get_all_files(arguments): + all_files = [] + + for root, directories, files in os.walk(arguments["input_path"]): + all_files += [os.path.join(root, f) for f in files if is_json(f)] + + return sorted(all_files) + +def is_json(filename): + return os.path.splitext(filename)[1] == ".json" + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +main() + + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/convert-librispeech.py b/platform/micro-services/data-export/source/convert-librispeech.py new file mode 100644 index 00000000..293168db --- /dev/null +++ b/platform/micro-services/data-export/source/convert-librispeech.py @@ -0,0 +1,108 @@ + +from argparse import ArgumentParser +import logging +import csv +import json +import os +import shutil + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("This program converts the LibriSpeech " + "data format to the default CSV audio file format.") + + parser.add_argument("-i", "--input-path", default = "", + help = "The path to the LibriSpeech dataset.") + parser.add_argument("-o", "--output-path", default = "", + help = "The output path to save the dataset.") + parser.add_argument("-v", "--verbose", default = False, + action="store_true", + help = "Set the log level to debug, " + "printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + convert_librispeech_to_csv(arguments) + +def convert_librispeech_to_csv(arguments): + + convert_to_csv("dev-clean", arguments) + convert_to_csv("dev-other", arguments) + convert_to_csv("test-clean", arguments) + convert_to_csv("test-other", arguments) + convert_to_csv("train-clean-100", arguments) + convert_to_csv("train-clean-360", arguments) + convert_to_csv("train-other-500", arguments) + +def convert_to_csv(name, arguments): + + path = os.path.join(arguments["input_path"], name) + + all_files = get_all_files(path) + + if not os.path.exists(arguments["output_path"]): + os.makedirs(arguments["output_path"]) + + csv_path = os.path.join(arguments["output_path"], name + ".csv") + + with open(csv_path, "w") as output_file: + writer = csv.writer(output_file, delimiter=',', quotechar='"') + + for filename in all_files: + base_directory = os.path.dirname(filename) + + with open(filename, "r") as label_file: + for line in label_file: + tokens = line.split(" ") + audio_file = os.path.join(base_directory, tokens[0] + ".flac") + label = " ".join(tokens[1:]).strip() + + writer.writerow([audio_file, label]) + +def is_txt(filename): + return os.path.splitext(filename)[1] == ".txt" + +def get_all_files(path): + all_files = [] + + for root, directories, files in os.walk(path): + all_files += [os.path.join(root, f) for f in files if is_txt(f)] + + return sorted(all_files) + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s -' + ' %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +main() + + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/convert-librivox-google-cloud.py b/platform/micro-services/data-export/source/convert-librivox-google-cloud.py new file mode 100644 index 00000000..df8cfc84 --- /dev/null +++ b/platform/micro-services/data-export/source/convert-librivox-google-cloud.py @@ -0,0 +1,392 @@ +from argparse import ArgumentParser +import logging +import csv +import json +import os +import queue +import threading +import hashlib +import gc +from google.cloud import storage +from smart_open import open + +from pydub import AudioSegment + +logger = logging.getLogger(__name__) + +storage_client = storage.Client() + +def main(): + parser = ArgumentParser("This program converts the DSAlign " + "data format to the default CSV audio file format.") + + parser.add_argument("-i", "--input-path", default = "gs://the-peoples-speech-aws-import/librivox-aligned/en", + help = "The GCP path to the DSAligned dataset.") + parser.add_argument("--max-count", default = 1e9, + help = "The maximum number of audio samples to extract.") + parser.add_argument("--cache-directory", default = "data", + help = "The local path to cache.") + parser.add_argument("-o", "--output-path", default = "gs://the-peoples-speech-aws-import/librivox-v0.2-1B", + help = "The output path to save the dataset.") + parser.add_argument("--worker-count", default = 4, + help = "The number of worker threads.") + parser.add_argument("-v", "--verbose", default = False, + action="store_true", + help = "Set the log level to debug, " + "printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + arguments["system"] = {"cache-directory" : arguments["cache_directory"]} + + setup_logger(arguments) + + convert_dsalign_to_csv(arguments) + + +def convert_dsalign_to_csv(arguments): + + directory = arguments["output_path"] + + logger.debug("Checking directory: " + directory) + if not os.path.exists(directory): + logger.debug("Making directory: " + directory) + os.makedirs(directory) + + with open(os.path.join(arguments["output_path"], "data.csv"), "w", newline="") as output_csv_file: + csv_writer = csv.writer(output_csv_file, delimiter=',', quotechar='"') + update_csv(arguments, csv_writer) + +def update_csv(arguments, csv_writer): + mp3_files = dict(get_mp3_files(arguments["input_path"], arguments), **get_mp3_files(arguments["output_path"], arguments)) + + total_count = 0 + + file_uploader = FileUploader(arguments) + + for bucket_name, file_name in get_all_object_paths(arguments): + if not is_audio(file_name): + continue + + mp3_path = os.path.join("gs://" + bucket_name, file_name) + + aligned_file_name = get_corresponding_align_file_name(file_name) + + mp3_size = get_blob_size(mp3_path) + + if mp3_size > 250e6: + logger.debug("Skipping mp3 from " + mp3_path + " with " + str(mp3_size / 1e6) + "MB which is too big") + delete_cached_file(arguments, "gs://" + os.path.join(bucket_name, aligned_file_name)) + continue + + logger.debug("Extracting alignments from " + str(aligned_file_name) + ", " + str(file_name)) + + alignments = load_alignments(bucket_name, aligned_file_name, arguments) + + if all_alignments_exist(arguments, mp3_path, alignments, mp3_files): + logger.debug("Skipping mp3 from " + mp3_path + " which is already fully converted.") + delete_cached_file(arguments, "gs://" + os.path.join(bucket_name, aligned_file_name)) + continue + + mp3 = load_audio(mp3_path, arguments) + + for entry in alignments: + start = entry["start"] + end = entry["end"] + + text = entry["aligned"] + + save_training_sample(mp3_files, file_uploader, csv_writer, mp3, mp3_path, start, end, text, entry, arguments, total_count) + + total_count += 1 + + if total_count >= int(arguments["max_count"]): + break + + del alignments + + delete_audio(mp3, mp3_path, aligned_file_name, arguments) + + if total_count >= int(arguments["max_count"]): + break + + file_uploader.join() + +def all_alignments_exist(arguments, mp3_path, alignments, mp3_files): + for entry in alignments: + start = entry["start"] + end = entry["end"] + + path = get_output_path(arguments, mp3_path, start, end) + + if not blob_exists(mp3_files, path): + return False + + return True + +def get_mp3_files(audio_path, arguments): + logger.debug("Getting MP3 files under " + audio_path) + + # Note: Client.list_blobs requires at least package version 1.17.0. + bucket_name, prefix = get_bucket_and_prefix(audio_path) + blobs = storage_client.list_blobs(bucket_name, prefix=prefix) + + mp3_files = {} + + for blob in blobs: + if is_audio(blob.name): + path = os.path.join("gs://" + bucket_name, blob.name) + mp3_files[path] = get_key(blob.name) + if len(mp3_files) > int(arguments["max_count"]): + break + + logger.debug(" Found " + str(len(mp3_files)) + " mp3 files") + + return mp3_files + +def get_key(path): + parts = split_all(path) + return os.path.splitext(parts[-2] + "-" + parts[-1])[0] + +def is_audio(path): + return os.path.splitext(path)[1] == '.mp3' or os.path.splitext(path)[1] == '.wav' + +def get_corresponding_align_file_name(path): + return os.path.splitext(path)[0] + ".aligned" + +def blob_exists(paths, path): + return path in paths + +def get_blob_size(path): + bucket_name, prefix = get_bucket_and_prefix(path) + try: + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.get_blob(prefix) + return blob.size + except: + return 1e9 + +def load_alignments(bucket_name, path, arguments): + local_cache = LocalFileCache(arguments, "gs://" + os.path.join(bucket_name, path)).get_path() + with open(local_cache) as json_file: + return json.load(json_file) + +def load_audio(path, arguments): + return MP3File(path, arguments) + +class MP3File: + def __init__(self, path, arguments): + self.path = path + self.arguments = arguments + self.mp3 = None + + def get(self): + if self.mp3 is None: + local_cache = LocalFileCache(self.arguments, self.path).get_path() + + self.mp3 = AudioSegment.from_mp3(local_cache) + + return self.mp3 + +def delete_audio(mp3, mp3_path, alignment_file_name, arguments): + del mp3 + gc.collect() + delete_cached_file(arguments, mp3_path) + delete_cached_file(arguments, alignment_file_name) + +def delete_cached_file(arguments, path): + local_cache = LocalFileCache(arguments, path, create=False).get_path() + logger.debug("Deleting cache file " + local_cache) + if os.path.exists(local_cache): + os.remove(local_cache) + +def extract_audio(audio, start, end): + return audio.get()[start:end] + +def save_training_sample(mp3s, file_uploader, csv_writer, audio, input_path, start, end, text, entry, arguments, total_count): + path = get_output_path(arguments, input_path, start, end) + + if not blob_exists(mp3s, path): + local_path = get_local_path(arguments, input_path, start, end) + + directory = os.path.dirname(local_path) + + if not os.path.exists(directory): + os.makedirs(directory) + + logger.debug("Saving sample: " + path + " at local path " + local_path) + + audio_segment = extract_audio(audio, start, end) + + audio_segment.export(local_path, format="wav") + + file_uploader.upload(path, local_path) + + csv_writer.writerow([path, text, json.dumps(entry)]) + +def get_output_path(arguments, input_path, start, end): + return os.path.join(arguments["output_path"], "data", "audio-" + hash_function(input_path + "-" + str(start) + "-" + str(end)) + ".wav") + +def hash_function(data): + return hashlib.sha256(data.encode('utf-8')).hexdigest() + +def get_local_path(arguments, input_path, start, end): + bucket, key = get_bucket_and_prefix(get_output_path(arguments, input_path, start, end)) + return os.path.join(arguments["system"]["cache-directory"], key) + +def get_all_object_paths(arguments): + + bucket_name, prefix = get_bucket_and_prefix(arguments["input_path"]) + blobs = storage_client.list_blobs(bucket_name, prefix=prefix) + + for blob in blobs: + yield bucket_name, blob.name + +def get_bucket_and_prefix(path): + parts = split_all(path[5:]) + + return parts[0], os.path.join(*parts[1:]) + +def split_all(path): + allparts = [] + while True: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s -' + ' %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +class FileUploader: + def __init__(self, arguments): + self.queue = queue.Queue(maxsize=512) + self.threads = [] + + for i in range(int(arguments["worker_count"])): + thread = threading.Thread(target=upload_files_worker, args=(self.queue,)) + thread.start() + self.threads.append(thread) + + def upload(self, path, local_path): + self.queue.put((path, local_path, False)) + + def join(self): + + for thread in self.threads: + self.queue.put((None, None, True)) + + for thread in self.threads: + thread.join() + +def upload_files_worker(queue): + while True: + path, local_path, is_finished = queue.get() + + if is_finished: + break + + logger.debug("Uploading " + local_path + " to " + path) + + try: + bucket_name, key = get_bucket_and_prefix(path) + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(key) + blob.upload_from_filename(local_path) + except: + pass + + os.remove(local_path) + +class LocalFileCache: + """ Supports caching. Currently it supports read-only access to GCS. + """ + + def __init__(self, config, remote_path, create=True): + self.config = config + self.remote_path = remote_path + + self.local_path = self.compute_local_path() + + if create: + self.download_if_remote() + + def get_path(self): + return self.local_path + + def download_if_remote(self): + if not self.is_remote_path(self.remote_path): + return + + self.download() + + def download(self): + if os.path.exists(self.get_path()): + logger.info(" using cached file '" + self.get_path() + "'") + return + + directory = os.path.dirname(self.get_path()) + + os.makedirs(directory, exist_ok=True) + + bucket, key = get_bucket_and_prefix(self.remote_path) + + logger.info( + "Downloading '" + self.remote_path + "' to '" + self.get_path() + "'" + ) + + bucket = storage_client.get_bucket(bucket) + blob = bucket.get_blob(key) + + blob.download_to_filename(self.local_path) + + def is_remote_path(self, path): + return path.find("gs:") == 0 + + def compute_local_path(self): + if not self.is_remote_path(self.remote_path): + return self.remote_path + bucket, key = get_bucket_and_prefix(self.remote_path) + return os.path.join(self.config["system"]["cache-directory"], key) + + + + +main() + + + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/convert-librivox.py b/platform/micro-services/data-export/source/convert-librivox.py new file mode 100644 index 00000000..57101ebf --- /dev/null +++ b/platform/micro-services/data-export/source/convert-librivox.py @@ -0,0 +1,253 @@ +from argparse import ArgumentParser +import logging +import csv +import json +import os +import boto3 +import botocore +import urllib.parse + +from pydub import AudioSegment + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("This program converts the DSAlign " + "data format to the default CSV audio file format.") + + parser.add_argument("-i", "--input-path", default = "", + help = "The S3 path to the DSAligned dataset.") + parser.add_argument("--max-count", default = 1e9, + help = "The maximum number of audio samples to extract.") + parser.add_argument("--cache-directory", default = "data", + help = "The local path to cache.") + parser.add_argument("-o", "--output-path", default = "", + help = "The output path to save the dataset.") + parser.add_argument("-v", "--verbose", default = False, + action="store_true", + help = "Set the log level to debug, " + "printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + arguments["system"] = {"cache-directory" : arguments["cache_directory"]} + + setup_logger(arguments) + + convert_dsalign_to_csv(arguments) + + +def convert_dsalign_to_csv(arguments): + + directory = arguments["output_path"] + + logger.debug("Checking directory: " + directory) + if not os.path.exists(directory): + logger.debug("Making directory: " + directory) + os.makedirs(directory) + + with open(os.path.join(arguments["output_path"], "data.csv"), "w", newline="") as output_csv_file, \ + open(os.path.join(arguments["output_path"], "metadata.csv"), "w", newline="") as metadata_csv_file: + csv_writer = csv.writer(output_csv_file, delimiter=',', quotechar='"') + metadata_writer = csv.writer(metadata_csv_file, delimiter=',', quotechar='"') + update_csv(arguments, csv_writer, metadata_writer) + +def update_csv(arguments, csv_writer, metadata_writer): + + total_count = 0 + + for bucket_name, file_name in get_all_object_paths(arguments): + if not is_audio(file_name): + continue + + aligned_file_name = get_corresponding_align_file_name(file_name) + + if not exists(bucket_name, aligned_file_name): + continue + + logger.debug("Extracting alignments from " + str(aligned_file_name) + ", " + str(file_name)) + + alignment = load_alignment(bucket_name, aligned_file_name, arguments) + + audio = load_audio(bucket_name, file_name, arguments) + + for entry in alignment: + start = entry["start"] + end = entry["end"] + + text = entry["aligned"] + + audio_segment = extract_audio(audio, start, end) + + save_training_sample(csv_writer, metadata_writer, audio_segment, text, entry, arguments, total_count) + + total_count += 1 + + if total_count >= int(arguments["max_count"]): + return + +def is_audio(path): + return os.path.splitext(path)[1] == '.mp3' + +def get_corresponding_align_file_name(path): + return os.path.splitext(path)[0] + ".aligned" + +def exists(bucket_name, path): + + client = boto3.resource('s3') + try: + client.Object(bucket_name, path).load() + except botocore.exceptions.ClientError as e: + return False + + return True + +def load_alignment(bucket_name, path, arguments): + local_cache = LocalFileCache(arguments, "s3://" + os.path.join(bucket_name, path)).get_path() + with open(local_cache) as json_file: + return json.load(json_file) + +def load_audio(bucket_name, path, arguments): + local_cache = LocalFileCache(arguments, "s3://" + os.path.join(bucket_name, path)).get_path() + + return AudioSegment.from_mp3(local_cache) + +def extract_audio(audio, start, end): + return audio[start:end] + +def save_training_sample(csv_writer, metadata_writer, audio_segment, text, entry, arguments, total_count): + path = get_output_path(arguments, total_count) + + directory = os.path.dirname(path) + + if not os.path.exists(directory): + os.makedirs(directory) + + logger.debug("Saving sample: " + path) + + audio_segment.export(path, format="mp3") + + csv_writer.writerow([path, text]) + metadata_writer.writerow([path, json.dumps(entry)]) + +def get_output_path(arguments, total_count): + return os.path.join(arguments["output_path"], "data", str(total_count) + ".mp3") + +def get_all_object_paths(arguments): + + client = boto3.client('s3') + paginator = client.get_paginator('list_objects_v2') + + bucket_name, prefix_name = get_bucket_and_prefix_name(arguments["input_path"]) + + page_iterator = paginator.paginate(Bucket = bucket_name, Prefix = prefix_name) + for page in page_iterator: + logger.debug("Iterating through page with " + str(page["KeyCount"]) + " objects") + + for page_object in page['Contents']: + yield bucket_name, page_object['Key'] + +def get_bucket_and_prefix_name(path): + result = urllib.parse.urlparse(path, allow_fragments=False) + + return result.netloc, result.path.lstrip("/") + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s -' + ' %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + + + + +class LocalFileCache: + """ Supports caching. Currently it supports read-only access to S3. + + TODO: match the MD5 hash of the object against S3 before downloading again + """ + + def __init__(self, config, remote_path): + self.config = config + self.remote_path = remote_path + + self.download_if_remote() + + def get_path(self): + return self.local_path + + def download_if_remote(self): + if not self.is_remote_path(self.remote_path): + self.local_path = self.remote_path + return + + self.local_path = self.compute_local_path() + + self.download() + + def download(self): + if os.path.exists(self.get_path()): + logger.info(" using cached file '" + self.get_path() + "'") + return + + directory = os.path.dirname(self.get_path()) + + os.makedirs(directory, exist_ok=True) + + s3 = boto3.client("s3") + + bucket, key = self.get_bucket_and_key() + + logger.info( + "Downloading '" + self.remote_path + "' to '" + self.get_path() + "'" + ) + + s3.download_file(bucket, key, self.get_path()) + + def is_remote_path(self, path): + return path.find("s3:") == 0 + + def compute_local_path(self): + bucket, key = self.get_bucket_and_key() + if key.endswith(".engine") or key.endswith(".mp3"): + return self.compute_s3_resource_local_path(key) + return os.path.join(self.config["system"]["cache-directory"], key) + + def compute_s3_resource_local_path(self, key): + repo_dir = "/".join(os.path.abspath(__file__).split("/")[:-3]) + path = os.path.join(repo_dir, "models", key) + return path + + def get_bucket_and_key(self): + return self.remote_path.split("/", 2)[-1].split("/", 1) + + + +main() + + + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/convert-timit-dataset-format.py b/platform/micro-services/data-export/source/convert-timit-dataset-format.py new file mode 100644 index 00000000..2ea4142e --- /dev/null +++ b/platform/micro-services/data-export/source/convert-timit-dataset-format.py @@ -0,0 +1,109 @@ + +from argparse import ArgumentParser +import logging +import csv +import json +import os +import shutil + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("This program converts the TIMIT data format to " + "the default CSV audio file format.") + + parser.add_argument("-i", "--input-path", default = "", + help = "The path to the TIMIT dataset.") + parser.add_argument("-o", "--output-path", default = "", + help = "The output path to save the dataset.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + convert_timit_to_csv(arguments) + +def convert_timit_to_csv(arguments): + all_files = get_all_files(arguments) + + if not os.path.exists(arguments["output_path"]): + os.makedirs(arguments["output_path"]) + + train_csv_path = os.path.join(arguments["output_path"], "train.csv") + test_csv_path = os.path.join(arguments["output_path"], "test.csv") + + with open(train_csv_path, "w") as output_train_file, \ + open(test_csv_path, "w") as output_test_file: + train_writer = csv.writer(output_train_file, delimiter=',', quotechar='"') + test_writer = csv.writer(output_test_file, delimiter=',', quotechar='"') + + for index, filename in enumerate(all_files): + label_filename = os.path.splitext(filename)[0] + '.TXT' + + new_filename = os.path.join(arguments["output_path"], + str(index) + ".wav") + + shutil.copyfile(filename, new_filename) + logger.debug("Copying " + filename + " to " + new_filename) + + label = get_label(label_filename) + + if is_test(label_filename): + test_writer.writerow([new_filename, label]) + else: + train_writer.writerow([new_filename, label]) + +def is_test(filename): + return filename.find("TEST") != -1 + +def get_label(label_filename): + + with open(label_filename) as label_file: + tokens = label_file.read().split(" ") + + return " ".join(tokens[2:]).strip() + +def is_wav(filename): + return os.path.splitext(filename)[1] == ".WAV" + +def get_all_files(arguments): + all_files = [] + + for root, directories, files in os.walk(arguments["input_path"]): + all_files += [os.path.join(root, f) for f in files if is_wav(f)] + + return sorted(all_files) + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +main() + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/convert-voicery.py b/platform/micro-services/data-export/source/convert-voicery.py new file mode 100644 index 00000000..efe1eb15 --- /dev/null +++ b/platform/micro-services/data-export/source/convert-voicery.py @@ -0,0 +1,152 @@ +import concurrent.futures +from google.cloud import storage +from argparse import ArgumentParser +import logging +import csv +import os +import json +import random +from pydub import AudioSegment + +logger = logging.getLogger(__name__) + +from smart_open import open + +def main(): + parser = ArgumentParser("Creates CSV file for voicery samples.") + + parser.add_argument("--input-path", default = "gs://the-peoples-speech-aws-import/voicery", + help = "The input path to search for data.") + parser.add_argument("--output-path", default = "gs://the-peoples-speech-aws-import/voicery/data.csv", + help = "The output path to save the training dataset.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + make_csv(arguments) + +def make_csv(arguments): + samples = [] + + get_voicery_samples(samples, arguments["input_path"]) + + save_samples(samples, arguments["output_path"]) + +def get_voicery_samples(samples, input_path): + mp3_files = get_mp3_files(input_path) + + new_samples = [] + + # We can use a with statement to ensure threads are cleaned up promptly + with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor: + # Start the load operations and mark each future with its URL + future_to_transcript = {executor.submit(get_voicery_transcript, path): (path, name) for path, name in mp3_files.items()} + for future in concurrent.futures.as_completed(future_to_transcript): + path, name = future_to_transcript[future] + try: + transcript = future.result() + except Exception as exc: + print('%r generated an exception: %s' % (path, exc)) + #else: + # logger.debug("transcript is " + transcript) + + new_samples.append({"path" : path, "caption" : transcript, "metadata": {"speaker_id" : "voicery_" + name}}) + + if len(new_samples) % 1000 == 0: + logger.debug(" loaded " + str(len(new_samples)) + " transcripts") + + samples.extend(new_samples) + +storage_client = storage.Client() + +def get_voicery_transcript(path): + base = os.path.splitext(path)[0] + + normalized_path = base + ".normalized.txt" + + bucket_name, prefix = get_bucket_and_prefix(normalized_path) + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.get_blob(prefix) + + return blob.download_as_text().strip() + +def get_mp3_files(audio_path): + logger.debug("Getting MP3 files under " + audio_path) + + # Note: Client.list_blobs requires at least package version 1.17.0. + bucket_name, prefix = get_bucket_and_prefix(audio_path) + blobs = storage_client.list_blobs(bucket_name, prefix=prefix) + + mp3_files = {} + + for blob in blobs: + if is_mp3(blob.name): + path = os.path.join("gs://" + bucket_name, blob.name) + mp3_files[path] = get_key(blob.name) + + logger.debug(" Found " + str(len(mp3_files)) + " mp3 files") + + return mp3_files + +def get_key(path): + parts = split_all(path) + return os.path.splitext(parts[-2] + "-" + parts[-1])[0] + +def is_mp3(path): + return os.path.splitext(path)[1] == ".mp3" or os.path.splitext(path)[1] == ".wav" + +def get_bucket_and_prefix(path): + parts = split_all(path[5:]) + + return parts[0], os.path.join(*parts[1:]) + +def split_all(path): + allparts = [] + while True: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts + +def save_samples(samples, path): + with open(path, "w", newline="") as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"') + + for sample in samples: + writer.writerow([sample["path"], sample["caption"], json.dumps(sample["metadata"])]) + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +if __name__ == "__main__": + main() + + + + diff --git a/platform/micro-services/data-export/source/convert-warc-to-csv.py b/platform/micro-services/data-export/source/convert-warc-to-csv.py new file mode 100644 index 00000000..c9af8b33 --- /dev/null +++ b/platform/micro-services/data-export/source/convert-warc-to-csv.py @@ -0,0 +1,110 @@ + +from bs4 import BeautifulSoup + +from warcio.archiveiterator import ArchiveIterator + +import csv +import cld3 + +from argparse import ArgumentParser + +import logging + +logger = logging.getLogger(__name__) + +def main(): + + parser = ArgumentParser(description="Convert WARC archive to csv, one line per entry.") + + + parser.add_argument("-o", "--output-path", default="", + help = "The input path to read a warc from.") + + parser.add_argument("-c", "--count", default=1e4, + help = "How many websites to extract.") + parser.add_argument("-l", "--language", default="en", + help = "The language of the text to extract.") + + parser.add_argument("-i", "--input-path", default="", + help = "The output path the save csv to.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + convert_warc_to_csv(arguments) + +def convert_warc_to_csv(arguments): + counter = 0 + with open(arguments["input_path"], 'rb') as input_file, \ + open(arguments["output_path"], "w", newline='') as output_file: + + writer = csv.writer(output_file, delimiter=',', quotechar='"') + + for record in ArchiveIterator(input_file): + + if record.rec_type == 'response': + if record.http_headers.get_header('Content-Type') == 'text/html': + html = record.content_stream().read() + clean_text = clean_html(html) + + if len(clean_text) > 0: + language_prediction = cld3.get_language(clean_text) + if language_prediction.language == arguments["language"]: + writer.writerow([clean_text, language_prediction]) + counter += 1 + + if counter >= int(arguments["count"]): + return + + if counter % 100 == 0: + logger.info("Saved " + str(counter) + " websites") + + + +def clean_html(html): + soup = BeautifulSoup(html, features="html.parser", ) + + # kill all script and style elements + for script in soup(["script", "style"]): + script.extract() # rip it out + + # get text + text = soup.get_text() + + text = text.replace('\x00','') + + # break into lines and remove leading and trailing space on each + lines = (line.strip() for line in text.splitlines()) + # break multi-headlines into a line each + chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) + + return "\n".join([chunk for chunk in chunks if chunk]) + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + + +if __name__ == "__main__": + main() + + + diff --git a/platform/micro-services/data-export/source/count-duration.py b/platform/micro-services/data-export/source/count-duration.py new file mode 100644 index 00000000..e69de29b diff --git a/platform/micro-services/data-export/source/data_export/__init__.py b/platform/micro-services/data-export/source/data_export/__init__.py new file mode 100644 index 00000000..b0596091 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/__init__.py @@ -0,0 +1,5 @@ + +from data_export.api.save_dataset import save_dataset + + + diff --git a/platform/micro-services/data-export/source/data_export/api/save_dataset.py b/platform/micro-services/data-export/source/data_export/api/save_dataset.py new file mode 100644 index 00000000..1b4e7475 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/save_dataset.py @@ -0,0 +1,9 @@ + +import csv + +def save_dataset(arguments, dataset): + with open(arguments["output_dataset_path"], "w", newline="") as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"') + for item in dataset: + writer.writerow(item) + diff --git a/platform/micro-services/data-export/source/make-librivox-train-test-split.py b/platform/micro-services/data-export/source/make-librivox-train-test-split.py new file mode 100644 index 00000000..4763c02e --- /dev/null +++ b/platform/micro-services/data-export/source/make-librivox-train-test-split.py @@ -0,0 +1,123 @@ +from argparse import ArgumentParser + +import logging +import csv +import json +import os +import random + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("This program makes a training and validation dataset " + "based on speaker id from the processed output of DSAlign.") + + parser.add_argument("-i", "--input-path", default = "", + help = "The input path to the metadata file.") + parser.add_argument("-c", "--speaker-count", default = 100, + help = "The number of speakers to put in the test set.") + parser.add_argument("--cache-directory", default = "data", + help = "The local path to cache.") + parser.add_argument("-ot", "--output-training-path", default = "", + help = "The output path to save the training csv.") + parser.add_argument("-ov", "--output-validation-path", default = "", + help = "The output path to save the validation csv.") + parser.add_argument("-v", "--verbose", default = False, + action="store_true", + help = "Set the log level to debug, " + "printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + arguments["system"] = {"cache-directory" : arguments["cache_directory"]} + + setup_logger(arguments) + + make_training_and_test_set(arguments) + + +def make_training_and_test_set(arguments): + + data = load_data(arguments) + + speakers = group_by_speaker(data) + + generator = random.Random(42) + + generator.shuffle(speakers) + + test_speaker_count = min(int(arguments["speaker_count"]), len(speakers) - 1) + + training_speakers = speakers[test_speaker_count:] + test_speakers = speakers[:test_speaker_count] + + with open(arguments["output_training_path"], "w", newline="") as output_training_csv_file: + training_csv_writer = csv.writer(output_training_csv_file, delimiter=',', quotechar='"') + for data in training_speakers: + for path, item in data: + training_csv_writer.writerow([path, item["aligned"]]) + + with open(arguments["output_validation_path"], "w", newline="") as output_validation_csv_file: + validation_csv_writer = csv.writer(output_validation_csv_file, delimiter=',', quotechar='"') + for data in test_speakers: + for path, item in data: + validation_csv_writer.writerow([path, item["aligned"]]) + +def group_by_speaker(data): + speakers = {} + + for path, metadata in data: + speaker_id = metadata["meta"]["speaker"] + + if not speaker_id in speakers: + speakers[speaker_id] = [] + + speakers[speaker_id].append((path, metadata)) + + return [speakers[speaker_id] for speaker_id in speakers.keys()] + +def load_data(arguments): + samples = [] + with open(arguments["input_path"]) as input_file: + csv_reader = csv.reader(input_file, delimiter=',', quotechar='"') + + for path, data in csv_reader: + samples.append((path, json.loads(data))) + + return samples + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s -' + ' %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +main() + + + + + + + + + + + + + + diff --git a/platform/micro-services/data-export/source/make-peoples-speech-train-test-splits.py b/platform/micro-services/data-export/source/make-peoples-speech-train-test-splits.py new file mode 100644 index 00000000..eb4c2f68 --- /dev/null +++ b/platform/micro-services/data-export/source/make-peoples-speech-train-test-splits.py @@ -0,0 +1,215 @@ +import concurrent.futures +from argparse import ArgumentParser +import logging +import csv +import os +import json +import random +from pydub import AudioSegment + +logger = logging.getLogger(__name__) + +from smart_open import open + +def make_splits(arguments): + both_samples = [] + train_samples = [] + test_samples = [] + + get_voicery_samples(both_samples) + get_common_voice_train_samples(train_samples) + get_common_voice_test_samples(test_samples) + get_librispeech_train_samples(train_samples) + get_librispeech_test_samples(test_samples) + get_librivox_samples(both_samples) + get_cc_search_samples(both_samples) + + train, test, development = split_samples(arguments, + both_samples, train_samples, test_samples) + + save_samples(train, arguments["train_path"]) + save_samples(test, arguments["test_path"]) + save_samples(development, arguments["development_path"]) + +def get_common_voice_train_samples(samples): + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/common-voice/train-flac-clean.csv") + +def get_common_voice_test_samples(samples): + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/common-voice/test-flac.csv") + +def load_csv_samples(samples, csv_path): + logger.debug("Loading samples from " + csv_path) + new_samples = [] + with open(csv_path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + + for row in reader: + path, caption = row[0], row[1] + + metadata = {} + if len(row) >= 3: + if len(row[2]) > 0: + metadata = json.loads(row[2]) + + new_samples.append({"path" : path, "caption" : caption, "metadata" : metadata}) + + logger.info("Loaded " + str(len(new_samples)) + " from " + csv_path) + + samples.extend(new_samples) + +def get_librispeech_test_samples(samples): + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/librispeech-formatted/dev-clean.csv") + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/librispeech-formatted/dev-other.csv") + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/librispeech-formatted/test-clean.csv") + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/librispeech-formatted/test-other.csv") + +def get_librispeech_train_samples(samples): + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/librispeech-formatted/train-clean-100-clean-metadata.csv") + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/librispeech-formatted/train-clean-360-clean-metadata.csv") + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/librispeech-formatted/train-other-500-clean-metadata.csv") + +def get_librivox_samples(samples): + # 800 hours + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/librivox-v0.3-1B/data-flac.csv") + +def get_voicery_samples(samples): + # + load_csv_samples(samples, "gs://the-peoples-speech-aws-import/voicery/data-clean-durations-flac.csv") + +def get_cc_search_samples(samples): + # + load_csv_samples(samples, "gs://the-peoples-speech-west-europe/archive_org/v0.2/data-clean-duration-flac.csv") + +def split_samples(arguments, both_samples, train_samples, test_samples): + + # id -> sample + both_ids = group_samples_by_id(both_samples) + train_ids = group_samples_by_id(train_samples) + test_ids = group_samples_by_id(test_samples) + + # id -> sample + test_and_both_ids = join_ids(both_ids, test_ids) + + test_set_size = min(len(test_and_both_ids), int(arguments["test_set_size"])) + + # id -> sample + test = extract_samples(test_and_both_ids, test_set_size) + development = extract_samples(test_and_both_ids, test_set_size) + + remaining_both_ids = remove_samples(both_ids, test) + remaining_both_ids = remove_samples(remaining_both_ids, development) + + all_train_ids = join_ids(remaining_both_ids, train_ids) + + train = extract_samples(all_train_ids, len(all_train_ids)) + + return drop_id(train), drop_id(test), drop_id(development) + +def drop_id(dataset): + return [value for key, value in dataset] + +def join_ids(left, right): + return left + right + +def remove_samples(left, right): + ids = set([key for key, value in right]) + + return [(key, value) for key, value in left if not key in ids] + +def get_id_for_sample(id_map, sample): + return sample["path"] + +def group_samples_by_id(samples): + id_map = {} + + for sample in samples: + sample_id = get_id_for_sample(id_map, sample) + + if not sample_id in id_map: + id_map[sample_id] = [] + + id_map[sample_id].append(sample) + + ids = stable_shuffle(id_map) + + return ids + +def stable_shuffle(id_map): + id_list = [(key, value) for key, value in id_map.items()] + + generator = random.Random(42) + + generator.shuffle(id_list) + + return flatten_id_list(id_list) + +def flatten_id_list(id_list): + new_samples = [] + + for index, samples in id_list: + for sample in samples: + new_samples.append((index, sample)) + + return new_samples + +def extract_samples(ids, count): + sample_count = min(len(ids), count) + + new_samples = ids[:sample_count].copy() + + del ids[:sample_count] + + return new_samples + +def save_samples(samples, path): + with open(path, "w", newline="") as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"') + + for sample in samples: + writer.writerow([sample["path"], sample["caption"], json.dumps(sample["metadata"])]) + +def main(): + parser = ArgumentParser("Creates people's speech train, test, " + "development splits.") + + parser.add_argument("--train-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.9/train.csv", + help = "The output path to save the training dataset.") + parser.add_argument("--test-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.9/test.csv", + help = "The output path to save the test dataset.") + parser.add_argument("--development-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.9/development.csv", + help = "The output path to save the development dataset.") + parser.add_argument("--test-set-size", default = 3000, + help = "The number of samples to include in the test set.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + make_splits(arguments) + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +if __name__ == "__main__": + main() + + + diff --git a/platform/micro-services/data-export/source/make-tar-google-cloud.py b/platform/micro-services/data-export/source/make-tar-google-cloud.py new file mode 100644 index 00000000..5ab3c3d9 --- /dev/null +++ b/platform/micro-services/data-export/source/make-tar-google-cloud.py @@ -0,0 +1,186 @@ + +import concurrent.futures +from google.cloud import storage +from argparse import ArgumentParser +import logging +import csv +import os +import json +import tarfile +import io +from smart_open import open + +logger = logging.getLogger(__name__) + +storage_client = storage.Client() + +def main(): + parser = ArgumentParser("Takes all of the files in a dataset and uploads into a TAR.GZ archive .") + + parser.add_argument("-i", "--input-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.5/train.csv", + help = "The output path to load the dataset from.") + parser.add_argument("-o", "--output-path", default = "gs://the-peoples-speech-west-europe/peoples-speech-v0.5/train.tar.gz", + help = "The output path to save the test dataset.") + parser.add_argument("-v", "--verbose", default = False, action="store_true", + help = "Set the log level to debug, printing out detailed messages during execution.") + + arguments = vars(parser.parse_args()) + + setup_logger(arguments) + + make_tar_gz(arguments) + +def make_tar_gz(arguments): + samples = load_csv(arguments["input_path"]) + + updated_samples = update_samples(samples) + + writer = ArchiveWriter(arguments["output_path"], updated_samples) + + writer.run() + +def load_csv(csv_path): + new_samples = [] + with open(csv_path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + + for row in reader: + path, caption = row[0], row[1] + + metadata = {} + if len(row) >= 3: + metadata = json.loads(row[2]) + + yield {"path" : path, "caption" : caption, "metadata" : metadata} + +def update_samples(samples): + for sample in samples: + yield (update_path(sample["path"]), sample["path"], sample["caption"], sample["metadata"]) + +def update_path(path): + bucket_name, prefix = get_bucket_and_prefix(path) + return prefix + +def get_bucket_and_prefix(path): + parts = split_all(path[5:]) + + assert len(parts) > 1, str(parts) + + return parts[0], os.path.join(*parts[1:]) + +def split_all(path): + allparts = [] + while True: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts + +def open_archive(path): + logger.debug("Opening archive for writing, " + path) + tar_file = open(path, mode="wb", transport_params={"min_part_size" : (2**18 * 16)}) + return tarfile.TarFile(fileobj=tar_file, mode="w"), tar_file + +class ArchiveWriter: + def __init__(self, archive_path, samples): + self.archive, self.archive_file = open_archive(archive_path) + self.samples = samples + + self.csv_file_name = "data.csv" + self.csv_file = open(self.csv_file_name, newline="", mode="w") + self.csv_writer = csv.writer(self.csv_file, delimiter=',', quotechar='"') + + def run(self): + + sample_count = 0 + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + + while True: + sample_batch = self.get_next_batch() + + if len(sample_batch) == 0: + break + + # Start the load operations and mark each future with its URL + future_to_data = {executor.submit(load_file, path): (updated_path, path, transcript, metadata) for updated_path, path, transcript, metadata in sample_batch} + for future in concurrent.futures.as_completed(future_to_data): + updated_path, path, transcript, metadata = future_to_data[future] + try: + data = future.result() + info = tarfile.TarInfo(name=updated_path) + info.size = len(data) + data_buffer = io.BytesIO(data) + self.archive.addfile(info, data_buffer) + self.csv_writer.writerow([updated_path, transcript, json.dumps(metadata)]) + + logger.debug("loaded %s bytes from %s for sample %s " % (len(data), path, sample_count)) + + sample_count += 1 + + del data_buffer + del data + del info + del updated_path + del path + del transcript + del metadata + except Exception as exc: + print('%r generated an exception: %s' % (path, exc)) + + + self.csv_file.close() + + self.archive.add(self.csv_file_name) + self.archive.close() + self.archive_file.close() + + def get_next_batch(self): + batch = [] + for i in range(1024): + try: + batch.append(next(self.samples)) + except StopIteration: + break + + return batch + + +def load_file(path): + bucket_name, prefix = get_bucket_and_prefix(path) + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.get_blob(prefix) + + data = blob.download_as_bytes() + + return data + + +def setup_logger(arguments): + + if arguments["verbose"]: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + logger.addHandler(ch) + +if __name__ == "__main__": + main() + diff --git a/platform/micro-services/data-management/README.md b/platform/micro-services/data-management/README.md new file mode 100644 index 00000000..655e6206 --- /dev/null +++ b/platform/micro-services/data-management/README.md @@ -0,0 +1,13 @@ + +# Data Management + +Persistent storage, organization, and caching of speech data after it is ingested. + +# Usage + +peoples-speech data-management import dataset my_dataset.tar.gz + +# Principles + +* Data organized by organization -> project -> samples + diff --git a/platform/micro-services/deep-learning/README.md b/platform/micro-services/deep-learning/README.md new file mode 100644 index 00000000..95e3da6d --- /dev/null +++ b/platform/micro-services/deep-learning/README.md @@ -0,0 +1,23 @@ + +# Deep Learning + +Deep learning speech model training and inference. + +# Usage + +Train a speech to text model on the specified dataset. + + +``` +peoples-speech deep-learning train dataset model +``` + + +Evaluate a speech to text model on the specified dataset. + + +``` +peoples-speech deep-learning eval model dataset +``` + + diff --git a/platform/micro-services/forced-alignment/README.md b/platform/micro-services/forced-alignment/README.md new file mode 100644 index 00000000..b9954ac9 --- /dev/null +++ b/platform/micro-services/forced-alignment/README.md @@ -0,0 +1,10 @@ + +# Forced Alignment + +Force alignment of long speech and text data into shorter segments that match +with high confidence. + +# Usage + +peoples-speech forced-alignment unaligned-dataset aligned-dataset + diff --git a/platform/micro-services/forced-alignment/source/forced_alignment/api/__init__.py b/platform/micro-services/forced-alignment/source/forced_alignment/api/__init__.py new file mode 100644 index 00000000..46bcb184 --- /dev/null +++ b/platform/micro-services/forced-alignment/source/forced_alignment/api/__init__.py @@ -0,0 +1,6 @@ + +from forced_alignment.api.align_dataset import align_dataset + + + + diff --git a/platform/micro-services/forced-alignment/source/forced_alignment/api/align_dataset.py b/platform/micro-services/forced-alignment/source/forced_alignment/api/align_dataset.py new file mode 100644 index 00000000..30ccf2a4 --- /dev/null +++ b/platform/micro-services/forced-alignment/source/forced_alignment/api/align_dataset.py @@ -0,0 +1,16 @@ + +import audiofile + +def align_dataset(config, dataset): + return [align_item(config, item) for item in dataset ] + +def align_item(config, item): + label, audio = item[0], item[1] + + duration_seconds = audiofile.duration(audio) + + if duration_seconds < float(config["forced_alignment"]["max_duration_seconds"]): + return (label, audio) + item[2:] + + assert False, "Not implemented." + diff --git a/platform/micro-services/ingestion/source/__init__.py b/platform/micro-services/ingestion/source/__init__.py new file mode 100644 index 00000000..b28b04f6 --- /dev/null +++ b/platform/micro-services/ingestion/source/__init__.py @@ -0,0 +1,3 @@ + + + diff --git a/platform/micro-services/ingestion/source/ingestion/api/__init__.py b/platform/micro-services/ingestion/source/ingestion/api/__init__.py new file mode 100644 index 00000000..f677bbf0 --- /dev/null +++ b/platform/micro-services/ingestion/source/ingestion/api/__init__.py @@ -0,0 +1,3 @@ + +from ingestion.api.ingest_search_hits import ingest_search_hits + diff --git a/platform/micro-services/ingestion/source/ingestion/api/ingest_search_hits.py b/platform/micro-services/ingestion/source/ingestion/api/ingest_search_hits.py new file mode 100644 index 00000000..3ff60d67 --- /dev/null +++ b/platform/micro-services/ingestion/source/ingestion/api/ingest_search_hits.py @@ -0,0 +1,21 @@ + +from cloud_hal.api import is_path + +def ingest_search_hits(config, hits): + dataset = [] + + for hit in hits: + label, audio_path = hit + + audio_path = ingest_audio(audio_path) + + dataset.append((label, audio_path)) + + return dataset + +def ingest_audio(audio_path): + if is_path(audio_path): + return audio_path + + assert False, "Not implemented for " + audio_path + diff --git a/platform/micro-services/peoples-speech/source/peoples_speech/__init__.py b/platform/micro-services/peoples-speech/source/peoples_speech/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/platform/micro-services/peoples-speech/source/peoples_speech/data_book/__init__.py b/platform/micro-services/peoples-speech/source/peoples_speech/data_book/__init__.py new file mode 100644 index 00000000..424b5d4a --- /dev/null +++ b/platform/micro-services/peoples-speech/source/peoples_speech/data_book/__init__.py @@ -0,0 +1,3 @@ + +from data_book.api.load_databook import load_databook + diff --git a/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py b/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py new file mode 100644 index 00000000..ffea867e --- /dev/null +++ b/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py @@ -0,0 +1,3 @@ + +from data_export.api.save_dataset import save_dataset + diff --git a/platform/micro-services/peoples-speech/source/peoples_speech/task_manager/__init__.py b/platform/micro-services/peoples-speech/source/peoples_speech/task_manager/__init__.py new file mode 100644 index 00000000..1aab2d3f --- /dev/null +++ b/platform/micro-services/peoples-speech/source/peoples_speech/task_manager/__init__.py @@ -0,0 +1,2 @@ + +from task_manager.pipelines.make_dataset import make_dataset diff --git a/platform/micro-services/quality/README.md b/platform/micro-services/quality/README.md new file mode 100644 index 00000000..3844678f --- /dev/null +++ b/platform/micro-services/quality/README.md @@ -0,0 +1,11 @@ + +# Quality + +Detect and remove defects in data. + +# Usage + +peoples-speech quality remove-defects dataset cleaned-dataset + + + diff --git a/platform/micro-services/quality/source/quality/api/__init__.py b/platform/micro-services/quality/source/quality/api/__init__.py new file mode 100644 index 00000000..d7a19717 --- /dev/null +++ b/platform/micro-services/quality/source/quality/api/__init__.py @@ -0,0 +1,4 @@ + +from quality.api.clean_dataset import clean_dataset + + diff --git a/platform/micro-services/quality/source/quality/api/clean_dataset.py b/platform/micro-services/quality/source/quality/api/clean_dataset.py new file mode 100644 index 00000000..0dbd6f40 --- /dev/null +++ b/platform/micro-services/quality/source/quality/api/clean_dataset.py @@ -0,0 +1,5 @@ + +def clean_dataset(config, dataset): + return dataset + + diff --git a/platform/micro-services/search/README.md b/platform/micro-services/search/README.md new file mode 100644 index 00000000..9160f36e --- /dev/null +++ b/platform/micro-services/search/README.md @@ -0,0 +1,10 @@ + +# Search + +Search for relevant data from many sources. + +# Usage + +peoples-speech search data-book dataset + + diff --git a/platform/micro-services/search/source/search/__init__.py b/platform/micro-services/search/source/search/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/platform/micro-services/search/source/search/api/__init__.py b/platform/micro-services/search/source/search/api/__init__.py new file mode 100644 index 00000000..298d3fea --- /dev/null +++ b/platform/micro-services/search/source/search/api/__init__.py @@ -0,0 +1,3 @@ + +from search.api.search_for_data_book import search_for_data_book + diff --git a/platform/micro-services/search/source/search/api/search_for_data_book.py b/platform/micro-services/search/source/search/api/search_for_data_book.py new file mode 100644 index 00000000..b1c10686 --- /dev/null +++ b/platform/micro-services/search/source/search/api/search_for_data_book.py @@ -0,0 +1,5 @@ + +from search.engine.search_engine_factory import SearchEngineFactory + +def search_for_data_book(config, data_book): + return SearchEngineFactory(config).create().search_for_data_book(data_book) diff --git a/platform/micro-services/search/source/search/command-line-interface/speech-search.py b/platform/micro-services/search/source/search/command-line-interface/speech-search.py new file mode 100644 index 00000000..2aefe38d --- /dev/null +++ b/platform/micro-services/search/source/search/command-line-interface/speech-search.py @@ -0,0 +1,67 @@ + +from search.api import search_for_data_book +from search.util import save_dataset +from search.util import load_databook + +import os + +from argparse import ArgumentParser + +import logging + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("Search API exposed as a command line utility.") + + parser.add_argument("-i", "--data-book-path", default=sample_databook_path(), help="Path to data book to search for.") + parser.add_argument("--dataset-path", default=sample_dataset_path(), help="Path to dataset to search.") + parser.add_argument("--engine", default="ExistingDatasetSearchEngine", help="Which search engine to use.") + parser.add_argument("--samples-per-page", default=4, help="How many search results per databook page.") + parser.add_argument("-o", "--output-path", default="searched-dataset.csv", help="The path to save the new dataset.") + parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") + parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") + + arguments = vars(parser.parse_args()) + + setup_logging(arguments) + + data_book = load_databook(arguments["data_book_path"]) + + dataset = search_for_data_book(arguments, data_book) + + save_dataset(arguments, dataset) + +def sample_databook_path(): + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "test", "marathi-databook.csv") + +def sample_dataset_path(): + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "test", "test-dataset.csv") + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + +if __name__ == "__main__": + main() + + + + + diff --git a/platform/micro-services/search/source/search/engine/existing_dataset_search_engine.py b/platform/micro-services/search/source/search/engine/existing_dataset_search_engine.py new file mode 100644 index 00000000..0692ad95 --- /dev/null +++ b/platform/micro-services/search/source/search/engine/existing_dataset_search_engine.py @@ -0,0 +1,109 @@ + +import hashlib +import csv +import tempfile +import json +import os +import whoosh.index +import whoosh.fields +import whoosh.qparser + +import logging + +logger = logging.getLogger(__name__) + +class ExistingDatasetSearchEngine: + def __init__(self, config): + self.config = config + + def search_for_data_book(self, data_book): + # load dataset index + self.load_index() + + # load samples + results = [] + result_set = set() + + for page in data_book: + results_for_this_page = self.search_for_page(page) + results_for_this_page = dedup(results_for_this_page, result_set) + results.extend(results_for_this_page) + + return results + + def search_for_page(self, data_book_page): + + logger.debug("Searching for page: " + str(data_book_page)) + + # figure out how many results per databook page + results_per_data_book_page = int(self.config["search"]["samples_per_page"]) + + label = data_book_page[0] + + with self.index.searcher() as searcher: + query = whoosh.qparser.QueryParser("label", self.index.schema).parse(label) + results = searcher.search(query, limit=results_per_data_book_page) + results = [(hit["label"], hit["path"]) for hit in results] + logger.debug("Got search results: " + str(results)) + + return results + + def load_index(self): + self.index_path = tempfile.TemporaryDirectory() + + # Create the index + schema = whoosh.fields.Schema( + label=whoosh.fields.TEXT(stored=True), + path=whoosh.fields.ID(stored=True), + content=whoosh.fields.TEXT) + self.index = whoosh.index.create_in(self.index_path.name, schema) + + # Load datasets into it + writer = self.index.writer() + for dataset_path in self.get_dataset_paths(): + with open(dataset_path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + for item in reader: + label, path = check_paths(dataset_path, item[0], item[1]) + writer.add_document(label=label, path=path) + + writer.commit() + + def get_dataset_paths(self): + if self.config["search"]["dataset_paths"] == "test": + return [test_dataset_path()] + + return [self.config["search"]["dataset_paths"]] + +def test_dataset_path(): + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "test", "test-dataset.csv") + +def check_paths(dataset_path, label, path): + if not os.path.isabs(path): + path = os.path.join(os.path.dirname(dataset_path), path) + + assert os.path.exists(path), "Path does not exist: " + path + + return label, path + +def dedup(results_for_this_page, result_set): + deduped_results = [] + + for result in results_for_this_page: + result_hash = get_hash(result) + if not result_hash in result_set: + deduped_results.append(result) + + result_set.add(result_hash) + + return deduped_results + +def get_hash(dictionary): + """MD5 hash of a dictionary.""" + dhash = hashlib.md5() + # We need to sort arguments so {'a': 1, 'b': 2} is + # the same as {'b': 2, 'a': 1} + encoded = json.dumps(dictionary, sort_keys=True).encode() + dhash.update(encoded) + return dhash.hexdigest() + diff --git a/platform/micro-services/search/source/search/engine/search_engine_factory.py b/platform/micro-services/search/source/search/engine/search_engine_factory.py new file mode 100644 index 00000000..1a786352 --- /dev/null +++ b/platform/micro-services/search/source/search/engine/search_engine_factory.py @@ -0,0 +1,14 @@ + +from search.engine.existing_dataset_search_engine import ExistingDatasetSearchEngine + +class SearchEngineFactory: + def __init__(self, config): + self.config = config + + def create(self): + if self.config["search"]["engine"] == "ExistingDatasetSearchEngine": + return ExistingDatasetSearchEngine(self.config) + + assert False, "Could not find search engine " + self.config["search"]["engine"] + + diff --git a/platform/micro-services/search/source/search/util/__init__.py b/platform/micro-services/search/source/search/util/__init__.py new file mode 100644 index 00000000..c8c2d8de --- /dev/null +++ b/platform/micro-services/search/source/search/util/__init__.py @@ -0,0 +1,6 @@ + +from search.util.save_dataset import save_dataset +from search.util.load_databook import load_databook + + + diff --git a/platform/micro-services/search/source/search/util/load_databook.py b/platform/micro-services/search/source/search/util/load_databook.py new file mode 100644 index 00000000..6900a3a9 --- /dev/null +++ b/platform/micro-services/search/source/search/util/load_databook.py @@ -0,0 +1,10 @@ +import csv + +def load_databook(path): + with open(path) as csv_file: + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + return [row for row in reader] + + + + diff --git a/platform/micro-services/search/source/search/util/save_dataset.py b/platform/micro-services/search/source/search/util/save_dataset.py new file mode 100644 index 00000000..3248f3ad --- /dev/null +++ b/platform/micro-services/search/source/search/util/save_dataset.py @@ -0,0 +1,11 @@ + +import csv + +def save_dataset(arguments, dataset): + with open(arguments["output_path"], "w", newline="") as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"') + for item in dataset: + writer.writerow(item) + + + diff --git a/platform/micro-services/search/test/marathi-databook.csv b/platform/micro-services/search/test/marathi-databook.csv new file mode 100644 index 00000000..c70b634b --- /dev/null +++ b/platform/micro-services/search/test/marathi-databook.csv @@ -0,0 +1,16 @@ +आहे,mr/आहे.mp3 +आणि,mr/आणि.mp3 +या,mr/या.mp3 +व,mr/व.mp3 +हे,mr/हे.mp3 +एक,mr/एक.mp3 +हा,mr/हा.mp3 +नाही,mr/नाही.mp3 +ते,mr/ते.mp3 +आहेत,mr/आहेत.mp3 +होते,mr/होते.mp3 +तर,mr/तर.mp3 +the,mr/the.mp3 +ही,mr/ही.mp3 +पण,mr/पण.mp3 +असे,mr/असे.mp3 diff --git a/platform/micro-services/search/test/mr/the.mp3 b/platform/micro-services/search/test/mr/the.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..d3068d6a59d1fe8758dd44b7779b33e7b661dd8f GIT binary patch literal 2112 zcmcK3={s9#9|!OgOC+gUA~8`hRjLzZDq8he+BC6^+6E&nYKkPacI=%gr6^j|-dLic zVk>DosHLQjeW{@(hH6w*H%r=jC|!e`$ua+cH_z|Id2|21pZj~=*LA=(t3CjIDe*mk zuxiTzGUzy^0TTcL$k4-J@tOvoUf{sn2~HyRLTs|O(_*xdws+{B(LL&>NOKNy^=Hk- z&Tk`c_C?+&;Ba7f&f4oHm>9yjq_TFakZj%T4<<^-5kk7i-I!V4$gv$&#&5 z&*Db3P`LAO9OPrLLYh9T#2IK(_?{80CSx)fxBJ)dGC^xL`@$s-La(DTlXC0&^xh>) z>=)K&&Kh=#^u)74w!1|%`7NpjKGz7&t~)o2{IfsW5!zk&N+=&h7!84;8`Q^AAzu;e zNg~2#lmMQ;(o=d^v62=HIHYD>UV}NJa%A$Mtn_rop@%vV_>1=p^4Y7TtgPIuq40N` z?L5dUUpqJIHcv&^1@q&LsaC;K_o^W;71MWbx zUKy`vNzNhGNNd4yhy#awWh})t84wgPRTb*yQb)|+zww`p_rC8mTOHckFfh9jR^#9` zA1Not4)_0wNXleo<3nu3;?2>T{_&%NPx{|Ginz=fVSvx&OTdxt;rV$d$ z+`~ITaJ)C_=;R-@T=3Rm4W}1p4{DkpY^|m`e}Q}@W#0}qAn;_;NMO7~ejBqU%~>oB zitRGhtWcJgp*?Vv`bc5amg+@O-9(~m#*U#Tes=elBpn{ zg=Q?)etz_D^vU|Jhh{S8TdZ;16iM)xawA%9LI#@~e&hQ^qqr!wjb|8m*?Ep zm1}Gqf1pkrQ)FXv0P-nHzT8xRU%*W7)Xy&fGHxL-`+6{L1ib%kpFOe#V#g=IjV*5L zljrD|>tS3DvxAd~@MsH+^x%w6ab}%rpR>oLxDEsxH>pW8a4u4Xw947bgnUBZBCuvZ`M>Z@0rmH!u=FA7W3QBSwxW8rxecdM(~V|%NPAJ5dF-Jk$K;Kn2`?sGWo6Z1bjKqxHV zD!Vve?4F|l(@YiY#v7=zjf=IBzYG%s&ZPvSuQEE%=CdNsgj_jh4J+-sU%9``p+vB%q5S=mWV68TYP z)M=P^{L*X-Pw0tCN|S$$Ldo0dRds@^cXV!gOMXe~ovy#gTG>(Enz%bX_ZJ{j8!z|` z&4Q30r7ZNs1Km!{t;%4b2k#t%dY1WDp#4r|-7$xP60BQ3xRLKbakM^kz>r9!ZfpKs ohx~YSfNmFn>w|4OyR8ehmLNYB0HkdJ$!tHi?M=wP{eSfT0lG==KmY&$ literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\205\340\244\270\340\245\207.mp3" "b/platform/micro-services/search/test/mr/\340\244\205\340\244\270\340\245\207.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..62cf6732f806d3719edc05977dee8e878d9dec71 GIT binary patch literal 2688 zcmcK4do)yQ8vyVF&bm0Q&CNB z`HCW8M5<{hw<1xwbyCXFWvC8j-ZT5OTK#v{`p#P4^T*zQ?C1GC&)VyGUmzmgg|Plv zGZ2!}M1v^!Jfy%Hl7SFf9}sCoMIe-2F?CbDZDAH*)SLHz`pN*{Vw-yNo8lhBw1xPK zO#rmKIX4?aY=0p!v`vd@$1j;XLQ8}Bl>wUulOqXXKKByQV`$#gLpWs}=2_9r5wQ4|IFCwPHv0-5#y`O4hns0Tv(-K8$h_c%XuDSpr%l1I)#Hfu zgME_v7v5zdwm#ZT6`-M!JaXVxQ3+T8+kd{)>@&rvsRE#4x-8Rf>_A`BuR|6QXKkGC zr1gFl9KGMIKHLKPD!4yKcnEG8)KW}e+@xiT*I&%4UF}8lFAJT#?{%6ii_t#ya_hET zOu`M_31pB{n>apPLmacgN@Ud)ac3}nR2Y~U$NjQFXB`k+2Ow(74~VCF)&{ls(KBFQ zc^UIbE#j#!RIFM4(%$UYXZm!+HEj=3Wb>u|l^o;90+=>?ikpv z93rLr^MffI=Q37vScd#7d)ctru!8|fu&;)DsgMJ8b~WWLqXnkC+qQRt+*z<3L#}YB zOziG)soa0TWb-oga;qmh+4eM&Ax+DKnunj%J?SP;^=*EFbySqbefd`?$*PM`L# z{)whrHXQ=LfxqY1cwj__%!B!0oHGWOeT(_U4MD*VHH+Rkyy2bjVbpa@dV78JCUI{0 znJP6IQ5hHv+s|0dPjzi(^cO76{{BSvTGw$`nXh=o%m6S1hV{O_|MN5KD`Ari^B^RZ z|G@FP4Zzs&O!m%)cw?5mULL)k)jx5G*HQKRnoz1%)^mTQ`}wFX6=;^egoGRz1K zp;1lconq340@7NK$Xxruz&;kcUFt(rzT`KafI>=A(h@OntceK>F7ojHI^@aSXO2aO zs1rHFI@$jVl-g5>Q%S3KswVXW#6hp{i(8JFYwI>tc@@ z6+*~h{=n&x)7)6|B%F%i;;?Kb`@ava#8Qllj437)Yn1#(w;9OaRXn_gh>rHYfB24I z7A);+CEEK`k#5ZIGBiD-0|N5Gs1BEMaJK|BUN?T@g`O}_lYp4)lQ1&x0?k7uW% z%J*2FbAtW#*mZ^+2+86*$Jl;wUyc&G<(^`#HppPoY6zNV!tKnE=S^Z`dxH|PIGzel zvTdhBjLy79#?vddr05TVt6)BGyPl35r6*rpEL32=kfk4LNJ8BmU}JYLE=UWIWu`k&6-gMAl_zobmM|2MYkB*0Ay->sAj!Vb6$64Ewg+h zSTYil2AMrRuQG4L|$+LJgN zy+dwMcJBHr$|S?ByO1jBqVrkIC}|dG78AgG6ICtD)mw$JH#ppw=fzI-^NO&)8#8Rp zg9d(1pz-1k*N2#m#$Q~psr@|A4N>W*KCGZ+h20)srin0ov}1J+{%fjZhvM*%qm=w2 zx-+fZUzR!}aRoqpuHZ_`lS}zjQ2_f<8mj~xXoSt8ZVtR&UUcCe z^yCWn=+vmBiNwS*H*8iEcDv~c3A-8gBMdCDH0W7yQ>k<)AZg&$$1x5pxH8FdUPYa= zn44wPzzFq@%1r0ZH(^E7AW%V9cCBaPAtf2_5W6XK+hVRHK3=*;uR}om*{*x2oEef% z6l33kJ*9W1)nMOo#WI5w=w%n*A4|+WA<_y&xrBzFBJ=Rf2KJ$nq12@YD*7fm3W0dK z!+#?sUDC67JsqCp#hc9C4hGA#W-vIiEq8E zv9i zVicML$%@I7hy*o1w1FB{1pE3}C5t%dfltDe3VhVx1z5lGR=nUloe`>MgZ0sc`qe!` z-?fQ}10L{(FLXFP&AwZmzd-}B{<2=+xRUB#+Yd1tL?eis^1h+I3x#{FYJLi5cS%#l z27a(_jKLeZL3ce8)ZglNWjJ@NriJ1?xCkr|J(3UnbOuVChQY1jJi3Lx`~RH! MufD>5&HvH=3p#5$TL1t6 literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\206\340\244\243\340\244\277.mp3" "b/platform/micro-services/search/test/mr/\340\244\206\340\244\243\340\244\277.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..12fb99930d7f4edc803294c570b2d60f4f9e0ea0 GIT binary patch literal 2400 zcmcK5c{H2p9tZF@5h}qnB9u-qu~pF$OHImDXhIWvVyRXa)KW`L(OO!!L`iFHLbPJc zSZh$)VOlNisil!=lg!1-+7+%o)`Q` zZUYq48&?5Yaw7}Kj6h~a!ErEqYa1J2FXC+9)Q;CHTZvUdO(b|3{R8q??`LFTW^4&^ zNbHhT<`UiMBx2j}`ep;IL@F#PGh9%D#jn+R_|ywbO=OA(I`s)C@#QfNR{@wU8usJ` zea^sgvrl~9aPG+mLz-Z&TS(eX@9#bx^20wIyE`K$SIsB9``(z;_?K`HPI^M61Qo>r zx6*W;@RPS-!4%7U1OHa;=x@G4-tCU?3ab`#2IMQN=np5sSoweehf}xdT6fy{M2c#Y z$D@JY*iM@I^L29%h^vuJ0W&)&nuL(8hyrdH4l&KZ9SW?E2FFFN1vGQ{L=Xg<`Bl6x zD_0Wcy)%`JexkeF?jr4k{GG~&V%lIgc|gf)LUxwg<+oEO?5Fo7J=kx>Kfh@6@_<%F z`QbNPzZHq5%HN6B`an=LeZK>Yk6sIVGP$;qH}Gny$B4x33in(Rl7y8CpEj=2_czq7 z=*In@_DVlcJihxrGr4WEi`&sSCW$l0> zO7*K>Yi~Z%K<`KQqYgy<$PJ=5=+UtWULQ5Cva_6dEOGne1I1lLcT0EEx87WIGtBSUv-E+*;+_z zA#k>+`Hs?d%@y4Y@B3m;0X6JQhc_4!l;-eAQ1^U>>@~JyK~JwmEyx zuy2b;U;Nyu5KU|d;_@(<*;n-%nfOaOl{W^v9wNSNzr)pfF<$;qoS7KImky+*i-Rjv zteS_F0534UBg3tc{hM#b%?G?*hrrDS_7d`Bp*K`gy31 ziu6lCR<%%Rf8z!r*LeqH#4DWZdyW`VC;fi$hl9uH(Wp7_ASVsWH{hNa{pm zuun9W%HQ3S(%wPB1b9dTd%q)-_skAb)F@|{K8!;q2yVp><8&-i8Su|6+yld?s9job z`3Jp{mdRC+Z;7#u$OJ~|9P}m0#{+U16U0QXQlDzUNb)%T*u;n$Q*~LiL8!MUV$+OY z8}o$_DamH=IR~f08>6N>bP6cA$j+|LluqjU5);9*E%8YN!I7NuWXLBg#>Zp={dA7; z-VbI$GIalNzCZb3%O8bZ3AVmE)R#nwCCTP@EVrF@q&C=xWacAmoyZz)H z=O|gFMry$g{ALBc{zx?$lY=ilxm+XnG!H>O4NXm#_--6ieVl=@^f<$@=-C-a1eZob zP1*QZ4Qp4MkIEX?NvyBa0*90Q*9WLbC%?L0Nn@tgl2&uhxj^~N9UR^Lj)AiY55_PX zGL=?0s1{*Vo}U;>hoptS9eWNCbkwj z+N4qQkFJv1&&#?B-_R>EpH5MtQ)Pqp<@=Y4R=s>}om7+ZavjbC+*rtx_@S68W*jg= z=RnUbTW)s#=_tC35uCS7cJpPe+GWU8u&Ea<+o}^KH~A8Cwpm<#J%4PcTAYDmjIM0{ z9(im%GssERz&KwuiL4%w^SjLeIp)d)5#k^}3Zuyk2S!-izIyi_qkz;XV35LB(zCd6 z{E&W!zKmCK*@UF~74>A9=P4ZTEHTGAQL2xQZ)rVt8jp;Akcb~}Cm+>C;oauUt^yth zP&-lrjrx4(&D(y0((%xfz%(>}HRnQ{j0(XGC5I<7H2q3-72&1VkF)hB?uu`MW$x+} zMANR|-a&^D$|?+QSZJ!#dLr-2%Bunv-fy-$(ADB8d($V_yFq*joS(eO2POXp$@~oE zcR>VTES;l&5gzX*K(+M0?+?iC*vxQrCEhv=3P|`tvxzj_ZUGFj$7aSRnTe2BDr2)1 z+>vUVojnTXIJO1|juXX>+}(T-4Dal!DdO+XX4=<=&)tIj1Pszz6=-d{a3b5j28)%G z1jVg9lm6(-i@YnG=I&%{l6OLzg+u&`~L!_es2^2 literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\206\340\244\271\340\245\207.mp3" "b/platform/micro-services/search/test/mr/\340\244\206\340\244\271\340\245\207.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..61e86146e5c06114f5bb9db4cc88047e511d27de GIT binary patch literal 2688 zcmcK6c|2789tZFAvpkzV83s-`}6_`};lL-+7(a0hfra0IDVJGmL-& z06>%x;*Z@rYn+vWLw?*$X}6UFK|y3;3eIL3hZS5#Iv<+0wdn4;VvC~h?`9z*3%`hn zD8wWis3l1vqu4m%0dG20;JfkGdJsVzakQt`m1&%+)ZxH|tg`Xg}5 zx_my|f&&(;roYrBbxx#JOneb=_#IehqxsKP>Y0-Xvl?GPL32;g?!*@3ir^RRE+P5? zznMbJ)hRerQl{|%O&1QP{qz+GUDsgQ#lR>%V?hGGW4l-o9e?K4!Lz=BPV`n)E|xic z$vT+(Hc_%b^GKi#<=LC5ncDEJu>8D>yC!RW3$-Od!Qf2AqHhx@Jc5S89iU*=FcW-k zICui`w_ymBdf0F-Yz|G*l;;y16bzcq-Y}`JmWom4D*3JuX)R=F>VQg_u>(zAtfDcs zYesIWx(Z~iE@N}86VuFsW-WG2es(w%-Mu^%rndE6;m9HiAJP%%F@=Tv?dWf(>H!KW zhgMVvlUKdnyHaJW!hF&RFIT@tCrL!JN~iRVzU`e1$v=9osuKh^Z8l3!IjIrT)2}6l z*$4eb_K9A=u-v;}|79s*J5fnf`H*>%+=4S2IP6vm`Em-`C8a=E1{1?BAiW%KJY6i+ z+aHk_Xyrrh%qToO8Sid+LeB#B@x9Shbi`N6_pp0a??y=n#d#P+r+@AqfludURBx#3 z=&R)*Ve^aW1N;|aRt5{P1hX11$d{CBF0TT@oS9DH%m^#mVB*+5_GgyF)l;d{p_k7! z!bPr(4||R9r6S<3r|!)Y-cNAC?GE&}@L_eJ_P95wHuVaUo8>fWB`t`l+1LQW8>L$5}x zQ^YT-uu*fFm-nvlFF5KLkze5mNw%jS-D&JOX#;iWot4csYzt)5CZ4J)F|I(qG#YlO z9{7EP+ThmG0CRPmnLxsE>kU^*DbH2gmRvh64~fbd?~@F2AnAo#ir+NHfjt{!FI^3h zV^;Ehn9DofgR_IhdwHwb#8!HCz=hWlXWF9lU%Jn^+;Yf;{GGBaQYldIlKDMO!h7=h z_?SQT(;R+QG>8)C zk`HefsdPiMMBu(|2-&rVfBzEbeb?b8=Tt`=PZR`errceL35!KcpJ7uOXQ|u7*;KoY z*N^s{$Km!jJsv27d;=6KKLuz%o>M0`6!-03@o?T~dDLn=S54eR%C)U7$Dg3{>@6tr!d!Ipa^GdOEAWv=Ux+RrOO$z+wEy?ivhERZVX@R}X+g_8wmjJy#RT^KNg<0xGxS#qbx+7<@D$QKaj* zCBG##O(}mvQxuI_URedVww`NML^HC-O7MGtDKzBA%IW@-0Su2YYi2DFpU+fqb4^lc zk=?n{yVjaa-gBN-^QMbzyF#Ix*km`F;8-^Ax_bWtk2ZM&HAL-(Wr!~;cl zxS|O`+{Z`w2v0`a-wirTLOxBtr-KH(%V%b1Eh+o=tK3Cp8PMi+GB5mCLE#!0by4Pf zLD217W1stn>l|T|jeJqWy&D7b@kC$kFssLtRSk6fcIN{jG#nR2u(PWZ2s0q)j#b{L z?hW~I@`pR(fw6pM)@UI=(A$~az354hl(%+-N;HHUHgRFn|kN_gQ=fzXe^j4Vb0!~Kn5e=Lc_ z)U+Iu9MgoSnMe_hk;^Z@wZIA`GYDV*Q>wZA&+HQ(rbyyj8|$4;?Bsjoj|{^Inw5r_ z;(!NtPw{(NH0nzU-XVAVXzBFNd4zs6nq?ILOk${;mu~2qRix`%;^btcG7N&%K%Vez z0Jj<|$;vr18HcFi!CI1`bx9@8^=pz;WbwM8wBJbR4VSxHHXxYi=$bq}eMyW7lLqcG z^Z@o5j-b9Anm<9-&)N~_@1i~`5^Bmy#JZ9bPlxC92yP_t#Y?r*#k`vus>$cRv-%o2 z!mp44WuIFbf{O95hiQ9$ue&jIn%$B@jp?#uq>Vp;Zt4o3pyGf?AJ zI=~AO)ydw}z}U#B@|PS(0>nXnfvlhTCUB{h`pax+!2kLc03hNZ{J8>tssHaKK=W7q O$NwdJ`CsJ!?f(n>D~z20 literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\206\340\244\271\340\245\207\340\244\244.mp3" "b/platform/micro-services/search/test/mr/\340\244\206\340\244\271\340\245\207\340\244\244.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..7826507d302033d419c6d66e87f9d04221e628d2 GIT binary patch literal 2880 zcmcK5XH-*Z8VB$jKnT?&BpP~^1*C)?fuSf2T{fbS)rf#JrMDAfUqC_Hh$tNdMDFIgXTQyv{jg`BkN5rG-}9XJ-t)fTt_}r2-HzJ=;5seIK(t&u zyjK~Z)1s4G1|(z;d)ok6DhAwrk{k(MyR5hoS*11-Sw?-MmK^CF@b2sp_j9%6?4e~d zoZQ_lUjD7UFs^2LmGCxVLTSo{QEM@m&mhfbMimC1z^))*`7D^BL}{bhjdE&Jt;Z9h zbnCYjpX2n_iFi+QcG*7+(;6hqdWjb+hbmo7SAywz+g5YkP2H|QHMfPk1B>?kN z{m>BPv!cszHHLmt8YECnv#{^$?IgGi}zr!>AP!R>N|* zq1i27e~o=1M&Q~f&$TMcPY$Hp*(P_jswsOS2b^z6qu}=?bOYGDzW;9GhmUc$sfSHd zVQHmaztkIaSqgpc5i237#471y`kf7e5IZ=-r|B~{I01c=*Z#mR_}~Kwz5u~>5R?Yj zFRJX5C)ZrpYU=E4SmU4UKV8c+1Xm6-(U%tFSS2AJj+U@11$?fi&4d=cyV>tDbmgC| z*8}SXZHKH6AN8NtUs{}ebFn`!S*j-cvs@hcE6!j1z5jf_S_Ot1sdN`7v^@eri~UHhVB=cB8jEAdXW^`|tA>0p@l1(+fl#*Tbde?$VZv6!pM$^O78huc zJT}xK*v&s<|84AC=H5V4)%=hgsz=XYFhZ}6U-C-oL`w&z_Xj;u2;%ht_k?fa4DS8p zBtgjMKtDCC20}Q}mQ?dAyR2;I2oVE3d&GaRE3^vx+Wb~cu3ei@h`BIoo_ zy^u`B-t*_>Qhq^?YZDXCYsG#EfLA;B>9&L5X*qbEo`kq@uT&_gylE`@hmYL8O7j`g zW|m%KvG^KWO0iq9CE<~2hxBK9x+P!xyr!DNVht>%5D(+{AU!{I;6AJ6e)0F&uRWo( zp^r6IUcU6>CP*%QEHsRxyj^uJI2520tKL~G&#Q%eE)?=A3CLwjTc329ze^o0%cBVU zg**xdIE>+BF|Ei{z z%iGI4t_Sj^kmeeZz(~P8d$J^UIIv>YdW>?R>+th{=}{H7`yPAy>8I5;b)jW$I)w~b z`}yOE;WMN8H1VL)#lUTcjVZ-fSBSKOOIg0_o~_!?lu99*^u<*84HAbNHzEHL5?LDt zjAW!Wzg)B`*WgxVe0S_|;!;q{)4Az{{$qh^smvFc=tqCm|3-W}2<}y91a#rt0605# zE4jsdIu>(2Q&`_hH0Z5q#_w}J)n^1~y^DS=VV=wx;DLNSq>wUg{zQT~3e&4@L|T;Z z5yziBCoav}+lgy@IQ%m$Z$!>gxwqYkfJZYY3?xMxWFX;>29do+IXLn5z3M~lHGf*@ z^hMzS*iS9hcDIXaTF_%EJuZTLYh<`;GBBD*;9t3d5S4s+cN5BLuGRGI_{!LFcIif@ z;&k{JEZjEL)!2>8*7NEbk#(mAsegvja!2I_WDsmrCnGk~Sa|1L`(D5nZi{xbl~jF! z7|6ekeEM@1FlIvlcfWDSbcI(V4Jbe~^FoH|8N}%gp9_L0QMgV$y9Cq5NyVhIZa&0r zn>b;<2OpN!_%%>ru&mXEOB%`p{NVu3#e{is-KPpXEnZj9=In-iA5_5QAO1pq(&dGC z#%aIvI&-P0k!4mn1>Puab1^odrRdcp`@qoC8GQlshnh71ThvlkdT%{38;h@0lx$@L z!u9lovjrLQHdunO_4KHUkGTru;jccB?~ne_mJH0=rL~6!dwRHM*xlCYMj|5>O_prA zt}JoY25}?ZP_ZLuQZ-|j$fB44e&vzlUW8XcZeSqMrXg42zKEsu*j-vvS{>X>wUf!^ z#YcE~<;sl>--G;L`Epr;fMv%P!~VS%5*IqlSSX?70k9BQDEBbJsiU~?G6sIp`O+j& zelXv|gxpFV9Z!r4Dhj@%E{~;_UKY{@bm?7Vu%pxmzH?0W6TP~RP#U!+*P)#|OImT_98A#akO6}4%h z-G|oAgBwvB1!Hhg*p4HqQ)t26+xxjvy$ZP5GW5|ZZL}K(`4U{0SN(vQoEFFCa&MP1<#oHnRr5hCzzZEY&6gyz!%AdiMd{3@|caHa$ zu6lv;Yg`jV$iILH5XJ%n6oN_Pu10JsZ*9u-_)&)08x(l_8EpO{P+dj)cyIIXJk;dt z9QWxF6_wxjQO1htsox!b^_>uVDQJyEliAmC1jZ1F? zulo2#N1obh^H1|c<&J4X%@TWWZ_|@mE_=u~;Ai=y152_6Hg%b;WAik;hXsZ)UuE08 zlPhxiZY@z;KybLJ>cI5q^z}r{tzxlEDZ`{}uRbH|=JIGo9cEaGew2ybIbZX5rzgEi z5=JMz@=-9l=x|c0mmv$}i=cppkFY)j(9A9Mz;LN!L~&cje$m!LSY4Q-(_uy1K8^i> zQh$D&wq+YNmfKj{L{~vZ4u`Y;`}%hd@?`*kULT;)+v1evfF|S#+jG(ofwcsw%gn8kO)ofAkkN71;|9nq%Sz~Ys3oW5 z`Jc}7p{%RR;wR}iOSWP+Bg7PKj71v%OSdh$;lY^p&(NNaM8VSl@C*|4n@Hf#R-`ba zwwN7j?Y}hkw{a$R67=00*{0`(tya)iBcxVkI^2uO!kX!!y6ck;X9apdgRjrMHd@_X zijvkviC*tUezEo#5xQN0IV0{2iI(_BapA&vkY5aZ+R!bxZx{8#VqynMP8}npXW^xn zz#>4YaFMvMp2zP^#deFNwhs?m#B844Z5qh?We6WB<>)Kg zxBzz?r;iIZ5__1ch|_$G-Q~+JYE^{XBH>2`Mw1miTf? z&6$nhco+b1x+CKN4^-JPie2bmoBBfk+bhu)JH#3xEjQi-;p|GTHW)fC|M`K-m7f{> zY7}lv|Cr}*_B_IhMzhLu`mOyjg7vni3#x=>Z=-p(Im|HbhkYN@oYn}8&T@xhlG!yA z0j=7fW-AhxyP)~#_w6RX21XT=vuEI`7CZ&CRD5z9yWG8VsN{b8aKqIydO=mbm$`|u zteiZW%@WZBDbN!?HQPMP+A1CnS2-25zaLBwf8Y1aGm&R`ZtgJCqM_|+i1Z@SJJ^o{ zW_pFVkgBN>X-stnr&Wn+=C#oq`xZK zJh2)Wo#G8oG8hNPr2FEPc-Nc`WR9v>9RkmZ=rN0K2E9j!^g!JmQfxyh5>KOaq_z(= zg;sYha*p`Ag$nu|b)0Z%f{9tU$1Q00^rw)-kkCtBGhqW85}$~^5>*Tci@D+!MXC9s z`9Q#rYpvvL}Opr*D9Qc4CXg6~j?(iHrF?Q&jQg~pr6 zVxvU*-gGZpAMtGyM}@hjVLe)PxoHyL5cS3<9T+L+y2b|o7|t_(Xj*BVUrDy9$LcHm zy!S+W{cI5@<53_g$3ln z0B^jU%kqAI3v{KmFEGxHVZgI&GX!B`hsgrC15HuJuw_g>4;K?~~j9trY`RSq%S4?4U2$%U7lS6;*t!AXXihC|?W9dC|7 zIW_Xm*&JOaQ^6mk>O@l~FyeK-9Bm9;A@TiCvAZ8YG7_9DkyZj8Q2UNUMcFpNhZSUu zmt~E_-D;3F;#SG`&w2@09rZKiasudu3+wSKwdSyNLbjJ{_EanM<{{HQ1EZtx2eI-- zEK?DRj@Z(R{glQmUENIL>T-9_KNeU zzg|+^rnTUbvoRh)qF10~ziP_&#jD%cdE;pcLrTzh^IJTb$NLeB9pRIum0N&4l5D$Xg;PWln38~7d?Ud0QD+4F_Cr3$$t+(l0 z*|*+0^RMC5)KBytZXKXO~}b_d}`0Bg%$l~ad%w|A(&G=(c}<~2h93Z3PeXWom`XFgE-@G z$N=Vm(8{|j7iegBE-zs@ZDPZ^Nch zL5alyd91F!mGSLsv{>YX7S337)i444<3#I_a@$wqMkDS&EY>`>{oI2B-lu#cQf^;I zgzDHUd^HV-=oX|T6hL*#o`vUB=sd25L?WT5?$ATEwTdS8#fn%M2VI4az1*o0ibz$>GBae;0P`rxu7H z6v!_@g>hG0j$U|js$=$bjAzQ={CIT!r+UfK+UPgb`KE94`*9MEV5VYDsX$v`geU(x zPB_Ggs*$)gzHSzRASh`M@BzWO==4tMRb2pxW7I2rB#!-K8^A{~PuD#$VOTS)9`xo7 zjK;jb&gHaEN8{%)u?+!PCL#W5tm)ek;UGA%xxT;n1pC> zT+leP^R@F9zH@^wpQ`^WMBnm*&r|rS%n?5XJS5e>r6iI#C%kFB5sVxHMW1;8G}XZ% zsN`^dWY;N2$zukVy-3ORA!fkkvy|MZs4G>AzLhB{7TVDXJ5#eUeD)nWgICvc3)-f4 zLCZ_wt80|(_=T!W25RbX3`}gbyL42twVts5^=1IW=JZW`_r(pi|MG(HB+{7&ZaW3h zNvk@~vRVt|{n_Ej-1QQ;%dOI~e2jvKQ| z-)Z7T&r#?;=e?}@-#3e5ZHa?(a^GzKXA1`RTaJb2o}m`}>H+VHe< z;brAqOML>=D~zX{QhQ_P?jW`!U*;+pY>b`(hhPV>$|7(Ql(Xpu{!^27%B)wM#0nSU z(!GY`If1+VtM4`o`K-njg-^mjjyC~6`_MSE?t8dAI&cY355HmDFaoZPV*4Cy@AO*P zH(44S;paSGq&b`o1Np!%bCud4R^r}GM(}8FjCL{l^tkuDiijI|C-w)CTHyD;*$C5k z+L-oKg>Q-p@~HuO$C#asNzY;G*+tU(obCQVl0EJ*9$i{H;xso|_U&vYSob(pnA8;4 z{h77Nw0DzBtO?;&MEwA({enkXd4D`Obl_xYX$XsJr)Ei+wsb{pPYe z(ag{7fC{wBD~%M=dOi90m%i*w`$)6T9*}+Bt>}fTEm|pjM|4SWJTT$O+|JjiW@T$? z9IYT^9=e3wWSwq6_z&wAN0^qTTZO&n)NB)aN4$uMB`c0ef3x{~{r86Oi0WhhugWvW ztfV>PHq?7TSvheJ%4Jhudr02Ip|s(n3f~i*=N1nP{>n7R(EwtR#;58OC7XHtBl@}J zD`op5#PWw7egzLF`s@wi4prS_@k58#AKvpY*RN{naP7CO3=SA}^sX(xtxlQnbX8JL z38u#=A;`_9SX@++tHKY!AlzJmo-t-w5cY0GQxikW8M=FZ;JD)(NNd1Y?JYxH)n?PF zydzTLJ&r7~`inNULGK*G_Pb!-5Z2w^!^z@+xtUqrU+cq;!l$AWkIe#Z?(qNC_Sa({ hz5b7X1pr{Kz)t;7{SQGD`Ro7fe{{BwZ~TAze*zk%B$faG literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\244\340\245\207.mp3" "b/platform/micro-services/search/test/mr/\340\244\244\340\245\207.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..0d9bfd3faa01255308d6f2419c7079458d2b149e GIT binary patch literal 2304 zcmcK2X;jl!769;n7A0W|36!!#_8pW2tg;D2f`k+>;8H*>!72(O21G_3Fa#o->_iBQ zmc4)=i?VqN0fRt_l^{ahP_ZnjilY!a3M6mxAJ3fm&~MY}`}EGc_x|oZ_dKvd9fCBL z`=k(t+Is|ohRG@9T*w51(zYiPkietZGc4Z&e8K|Bt}Ei4#W}M@^W8h~{XY~*P4_P5 zmyUa*P_D~_A6$tCl673ulwRzJkYKP$`yOIDHMz!FmOP;FafGP(bx4QWYfGB_lxRQ7 z_?Tq);d&4t_o>;QcX7Gg(V_DwY-_`@?ICpw$Zd!3R+=z|ZI4=qATvEp>HBSTsnz<0 zSRFlf)RBm=-{VRHDOIBY0P0@0yrBv9b?^?47zm06;(Gw9t8HC%>i2mYgI+&|=ajn7 zF&mSb8{P7ZhBx!xH+XkQBnpRf&0p}U={;#vc@GVZ>Umu9;+i_e=ZW8T4Nb(LY(u@1 z3Yv6o6MJ{G6L$diE%a|#zd__2hu$p{IQnwY%r!%w*-@g`0MCj&lfRaiPW@R-JW7`A z5Xk&QGq}_EVaqymHELteJoXdj02?fU<%~t*VtOe{++X8$I__KmD8Fp+Fza3-77P2z zS}2ozL@5J-Wl(V!x7YPWB9Ep$EU#^E z{?2~eDpM+`5&+QRAyX^_EV*Z&mCespUo!LNMngwOcbJ$Z!oJ!U6So==4>F)wDC%AC z`RqJv2Vs&U=ze`hQnAg_+K#_Dy&j4_jdh4_%cN0I>3;m7G*$jW(HOlBk1}7Ar!SmL zax|r$RJ3^I5d?K=dF+gO8$V#8rJ-RD`xq^3e=}5F#%oR}F8Xpn7lRarFS}}NpYm{( zvYR_D^_mbE=MCzFD}h7EAH1V{@W+K z?`nobp*%qxa!1*B_A2&EAK1T0pFFfxBDbH}P=QX|>~&hF(rt7gn{aV^2Zn!ez~DhBTJn!`+^$_H&c~;CVl|__K~RpPOCxa{{yX)O`*wkghhxK9 z3OS6)JcSA#bnr(m*#P-6J)GBClZ6&-2Gh<)4|Q|Q&U6%` z^`eyMjgBLB8!?Ny1(S*du+sFAH;uY;%)*Ch7la-0h5aoA1f37he=Re&u!Yw?dEHT# z9AuR2Qo~kMNcu=NQ9$M>)bNl0Dy)Fe~A z4KP^r3u-9IobOlJDzAZkXMBizJ~SD?qP(;WhQIOh!^jnzx7WK+kPX2iI*T@l%APlS zr%Ols^f(RZG24A$R&u^daxtB|b2M$^Z1C@Dw|Sukjp0twOrc=yQmhfSw5wYQ<6i^b1c%cU#j~bqvUZo0sGx9jKPiNC0j)eEU_Q$Ad=3t z%tJC?WZs_J_FP_6uLuIEwwdZ)y;!4{IejtW4Aq6=2{aA%1M!jWXQ3H&)~3=zlA42d zB#XmEE1~lfBQC~k!iB360~f2aAE^*7gPY&y3I+;eoLKtwyFxG&G~kqN$m*{p?zuRXC+6m6jDu=ANc9;+)jE zN1Vm+(`;}`@hqQNHhn#bjCx(GkNSyF(b;slMY}Q3OpU2e3s*jouD$nA-(*k9mtt&Z8;@GK1c(1-?$njg^MIDM16|;B zKYwXr4_LLa1oI?DWiH){$s8o+RX5pB?fDP}zy~T=zoUw!b&?pKsrj&1+5i9m literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\250\340\244\276\340\244\271\340\245\200.mp3" "b/platform/micro-services/search/test/mr/\340\244\250\340\244\276\340\244\271\340\245\200.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..7388eaaa2bc4822c8a5039279999fda43b584f90 GIT binary patch literal 2976 zcmcK5`#Y5B9|!Pf3>u7>8K;!9m2)!=k%}3NMbtP95_TMtQz|B;&dCW35=lZ0<^w&~mQ7oddQ?de4>~c9v4U3Xw7xn&&%0v#s@Bsy!;km&0< zb2;{EF15QXUhHOuokX8lT%pg{WG=TicMp7sCW3d_~wuBW}_k*YBKE_GsIV zw2=A{utjbO)M(gN(;(} zUCFWImHFSuAbp(}wWDgeGB7jzt&-<%Lo9U&g36<3rJ}7M$k$h)FZy{~b6G&aFjZDm z0prfFlSueBKaRb49&ue8FvVd&q+*ilo?e8oHLghnZ4bI_BrP6Q-+L&qgYHiJah~OSEkSJ04FgYF34z}Jc<0DEzkb3TifVfW zLEFErnoqL2_UTwK3#TDy2U5ZhyG7HEJ zBPshy)Gs{K3TwNCUEJvA>Gc?))6=q@e~YV7CzcD1&tAqsqczG>EyUFxK@-c0EDKA1 z81untA;Wu10k7>j@r;-}N-YESW#muxT?ZZug9AbhyyM;Tla&E&OjG9B{NE&v2*ED< z-0Y{~*SM-RZ~UPfe?)QSX~977UrzU!@n#OPC_}@k-Oy%t@5mE8=@8MmSa2iI`bB6N zs1zxh^&tLp{%g0|fvb+-<0EDfCox$@-}yBpE^%!o4pFS!+z0y8|Bi{8j_xnH{O9Lh z#Qo}q`kc10wJfi;yFI^uT&u81Qu{Qdt8$TDzNxe2C&%Bj|vzAEb{krmb63y2aDi+y^9gCxHhTgtqr0{oY`YqU($CA2QfXF+?IW`D2 z-?@t_cG2WHFTap4THLB~*w@Ffy*q(OY49^J%BgF6*Akz&b4T<E#YO`T|D`NQO0#8VPzn!-EB+5le zqp=)2ylUkzoqcvmBs^BT$T$Fs;I1u3C)7oGmI{~HJ9nqqIR1P>@Q`+fVE=neuSX6r z;RN>Gj-k<1Lmx>?72$gtE*s{bW2D^|4p?DXz^`uO;gr+U-T@gbWjUH#mD~6Pzf>S~ z&7FU_o3{h)+~Geh^m=IpL7T59`#zZ8{epn`LNH0&2~lKsKwK`}PJ{Vv=DNwjfSL}Hu>A!Z zvre@yvo<~JC=J&&j%<=t6QK3UyXRWNn&Pdb0a2;R4!mkjx3PgtKhq@H^~wWFMTbNW z*gu0U_oD&xS)iZ-+CG{D*xw{#Dxa!`p~bU)(q!p4O3O&oLDJJsYKUO!A!ts0tZg^lI63xvQLPCo74Anc!$KIstxEZBlMN_CRf z9c~j-M%?CBd_`1#dkX(Hl_6|{4F zq!S;~xM*N5Z4uII?O2u9>`%d3M;ZBS^d%1veptpT?YWgk`NipAhdvS!e|q**l*%1J zmk%tKv5fiqUTxUVkS*^E0bU#jfx5M_1G&A)l>7p_KqQ(l*j;_-nwm>ZU=IW}xzZH; zQzTNC{z%Qnq4#8W>9Mbujd)+hT8kBXacE3jcCO`Zo7BU>l|i4{VZl|Mz0Crt3G7HAZa&p>Qn`q5;OaQfoZh zWT8H_=?cB*hEQN+FygT)kt``xi+hlZhW&GLy6E6phWveACvz^X3*it>V-~Q z8PV0!2y^rV7CLU{?(*2GxMpDhw9@L@-E@GG$Ii**Te2j)FM?E$-J<#YsRf9{T# zM}?4Zye_CJrLZT~f+YClo8_iitZa@YckB=8qO;Bu5KCMGW=&kf?*H zj9rqQ`+{!kZOHKAa2V_dW7w`lU_K9g4(XiQ%7ws28utzKwNW>=XAhg|F??iGY8ygbvcLvUEEF4JdHg{~h_) Ymaty{07!d4u>Lju-vhAU@_+RI2ctu(U;qFB literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\252\340\244\243.mp3" "b/platform/micro-services/search/test/mr/\340\244\252\340\244\243.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..42428876fb9b302008c216bae31b15ea4775b7fd GIT binary patch literal 2016 zcmcK3eNfWb8VB$nVgzP}fj;}H`f-#jCGXorP{h`zNX=( z283zrOI|dy@})*?cLlS^EmqXj)?!^tGgE8}(l%1hjhdZ1vvdErJ9GE>@6375XTIl| zbAUGR7GzeM`x3$jc055k98OXBOyxpQ-qy@z_LINMuNUJckiN?4!^JsC{ldN?pK42G zPe=v7j)$)J?R+07X-MusRlK0W;j5RBigcFD*jRQl9X`b87)Xa0af#r)k2Ga;E$ICM zg>5UQ@$IF2U(Gw`2X9;2ren4{;YV3JwW&^Q5`N(9g~r~>5G~BoB`(aA$iguM*BrxK znz3Iov(9Chqap%a?cntFaXTv2uXjgoZ*3tFF6e!8GoukY%pVQ4f&K^prY?d-b4;2` zcqH~4rCQC*Ll~R(q@^v6m?u6Y5lTiLwKf;B4UzZ_2g*<@NU!h)C>89b`Dx1)l_w?* z(W`C7>NhV7D+RZjLQvhsdfx^=?x=zl#X}un!mh{kF1!+G=oMR024U@V0F>*z6h#K} zG`xQXmWnjY3s!;jfH{gA(Wg&uVW7b*r5)xzVe>7SQ84v2+zjH5dJ8}yl zTko4-M+W$?axr8WtQ^NaqmfwqBimc<8TuKI!2+T7<6O%#xqjxd^v7IQ+jD8T&?^sw5NsI&}cCBB7;0ZG`bhJvm;?|4U zLr2-KCL9dMOzcw=6#;7!h0c?mcL30BF3!C2dBAc>?`;m>>vJ43uO==-KdP7802oA6 zMLc|3_Zq{_JKG{*k0dr1Cv zGrMZqG^e~lL&j@qbIv&mG0>Zow_H5Hk0nl(3I4^l9=lA^W~n_fih^Q36l>SU`?T_d zk8$KRLEnZ5pY=_x)X{cPA!04rCu$ERCe{7SzIs^@b~yZq~O%^q&n z$-+MFoSQg&u(4)&IG(j=4|LgE^X{j8Y4o_+*c0@~_3Y4+~Uzkq;NoUoCo&L_3oIPWfgVJ3C+s@3$cy!H*-Og|^zJyT1Ct zfO9isIPbGF1x<+9oLzM{MYN4y`c-6|py_>_=?L>0Br+CQMOVeDC#`R;8HpczWQ{^C1joa^}AlW2bn6ayW^U}lbc1j?vs-0Zkj%f#YU%TRq*9%0{+ zEIqph?)(7mTf0u7KE3okr9D1=VDE2bftkY}-jA=D7b69FEEjx0U$m6`=F>BClH2+udHp zdH;z4cz3xP$yG_O|M-mVzl-X&$;H_@@=x_~L+?kg;&QOiFoXDCy8d+NFD(D#^B@Qw1ii2StN$^I SK7ZpM{$EJ3B+8w927*ix*B$3^GX+5 z9&S-Rj2f+D^LuSjDn&A z-o-^`!>e5O+27qE1w9O{fc!)9QJ>vmH-lkEc6vch3+E2PAZoYF-Qs}!{O(}S$@0Ux z^kgf|3~fzr_4EB=mO=;igp$Ts^C}2#H?SuY!gNwVnomNoa>mx6FT6m}GP~rIkO%?$ z;#tO&9psCn{VU>Nv!(z*>K7wSs2nYL`R}@(4)E{TljFKhljTY+xELud8&Yn*?7=i; zDLcSg?{|R!T0niIV);1{OgtO}_amKeMsCcQbmmsv476e+u5izP-H&WXMG~-xr%J+G zhhD>E%BV#QWz3)93f#Xttkx&C92_78vLkFv-V|Bog{ww-*~Fg8^*p3z4QCawqywxs z_`}Gx)%aFzsP50MPPO@3-OpE7y&m?}M=4a~0Bkqv6p(iJF4=0v&4?%d;H&-d!q(|a zvNy(c+U~CNE}9MAaCPq6)ia70_RGI4wA`I&>if~Zm7nQiSvni;*jU^it7i`MF9m88 zRmxc61gPPVFN6NQIR|)IMy+|6Aj)*?Jqv<7CX;!8)*RzLr7a*K&%;%{T2@@8ZT)BB zbN0Z0U42wrf1;q|Jj!*MsvN3;v}ugkaXtNbc{fqY5X z@zye6qKq1p*@xvmhA~*y$hox6yKb?o4NTnMUIBTHcQS zqub!dXo|X0poN}D>~q+)xj?bsMx>Iuc(Y{*5u`DscX3HT5Yc7@?MnKQBmE+>;5EZM zO=JukPM`J^VDCV_y!21@kAMgaS+t+i1W3t&xnhz8akwxc|&5M#)o!tfmH%ht* zuBVpXH&>F7JJ4eIWb)4mk6)107xr3D1UEV6XgXnh`)uXp!r|{~g#DgUM3JYR@?{S| zz6LVhz8#2ACc7V#$iNSeA+#F*aG0H1et_1Wh_4DIdBooyRQM2DQt+&PpYKhgj=+q?Ldt^) z00uBDw4$4x3q`p6Km;3_gb(3ZVOtvxa|7tI;sMlIXK1tDsLO`M{NK>aH$3S#6R7h(P#`!*y4VVGN&!h8uDFH z3dipN`55xSPmgOx3LLz!a6y~oiZ8>1>V$>{rsoE2A?B#`yS|7*yDNz;&z(2dH~CH$ zwJ(Q6ad^87jSTLEcKELYsgq|vC_iLADV!;F?e26(Z1+Px5v71H1BQgCZvZ7hk28U` zsTdaPsc)nB+F7z^xRJ-(HZ;dxskO5B*X<#U0LA6-7MGB)2C3tog&BDBIhCZ`_CEUrm`(B0LM25Mfpt*|I!e3WO4Dn$A#TiFb2DyXVn)vj7p6cfu~ruVCy zCW0L8V~`((5@|LCMi+1cYQi&L2&dfHvaVGO2IGBHRyk*3J|KP04QK82qk+!%dU_4t q9RDkU{3PT(S}-8b#QmfCZ)*hkc>o}I0^rBDmHeYD$fy4w{l5SY>Pm(H literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\265.mp3" "b/platform/micro-services/search/test/mr/\340\244\265.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..f552e8e786e2d2f193e5568dc5707cb26b71b86b GIT binary patch literal 2016 zcmcK3Yc$*0769;nG)htli6SKwRf5zoYU+6tB#1^zJx8nK72K%t=nLz%4NWnbRM9^z zDkT{4xSgRd=2DMFuWE_5blR@glzI)SL#rvPa&G=>`fcuqJ0I@;xX<3dv(MV=fOiZT zM2PJ82x&4p_>i{|eD^Cl14`J#C!y?jY+r$BpvXge^pn7hwx;=9cVxB(bI)zc%4MCP z{50!+jz0N9NY+xGa~qv}PB6|PjBh^0%jFU*CB-K0rvI|>Ee9$eN1$YAK@I{xYi{fk z)tP#odgtYS(Cl7jyl~`co!HemYLLo&Q^M7AZ)*;VFg#;!M)7k?D>K7r_*bPVu=s#3 z6iLUiJ@LZ78~LgEc_D`rH|2eGh-W>?O72H|Bho`L9U4c&hOhKFi+gtlMMS#qVtL)P z+{8o?co7AT1-eF=!3uzj0jf~?Q%%R*1ZL&4C#PyTWkI}ua163o+WP>QH#AV&>1#Eh zG4|aabGZR!Af)gu;$yecE25!sF?&esfu=Fsa^$2->h;-V)=S+t`SV3u0iPaNW~FcW zy0d;?n1%5Ip+8=6I`(R`opV}Ik3%4CV#2jIPLLx4U=@I=`3&vSYc#8jv*?-h5)CK_ z@wIeG!gy$AoL$~Q%_W_2#q+)_IPOEnzk8i=XsNLMrpukfT@89gan0heQw@IEX#JG5 zrod~zWp*u_U4*%u^QuRs-s+|=X9z_vZWwLBu$jJJ4g^$o+C>Yfh)*=YNYbF!G8o#< z&lzlbOO?>QW8yP|)JJ5#7Yk5z=#SVLCMPrX^+HHLZ5;r`Vr8wP8+krFc(5dybs%uS zC$Lasi}urV;KLHwI6S$M%MIP&THm~6y0U`!nz{sG5i}}*8;|<$)_kG!(9?Lu{s$7h z#QFlB34PtOYapwbY!%pz66vrflBySwEfnK3bewKC{_N4D>~F}&tGqy2*PfOI$07OL2ZbQ)s;=>1`i*MOyq(wPgS-+qvM$?+0w^_rgPnkJ%bi zQH}VzHrw*>q;rq|CVH>-itKm%inXbe>F^f=9tyzA>;AR%WgR|YJ=0IAPJ~k;G;X0^YtWE&4Drc? z<*QO?7!Awf8HoxKhG)7yRLWg_%l$(Iz}piF)a>0?`o)JOa#=Y3TkPK-*MN=DgN05W zTh(wDt4W!>^O^R2iFkvu!+R)ELd%M_=MlNK(urg-;#(11{9B-52K>?CYT$#t-B&)l z8`x%3e$v3Lf9B5i#%$6D;MML;8|yk4aT?oanY~O>*{U^q8$0HfXqtR!>3Zrx95d zm!bPIm>%ET>gBC$criK#M)kvNVF@8YP|JJC<~^?TdsGxY83_H36WUzbQpCB3WgN<5 z>i4c}Lm%{B#!*e|ZloaiBb<<5UP@E=Z>xT`bMt9{_Sx);Sn2IWG#ww zgs!*!Svlr{Ks2P6#NLsxf3t^O{t3@?hx*ZB!&cj)DS3mDbCheh5OYo_k@%*r>Qgt9 zKNu?`ry3i4ybEJ0<&p=1#fZP3(CU*3J;A2!R!bY?COcUq@YhHZhSw0KqFD6kZrs^> z!~}eG*N)f{|MM0L{m173xQ~ipoh`20oyDZn!%oYSQUO@AtA}dQGumxzYXS~eog%ju zN;`i*{6oa*LOL{T!p3eT3@n1*l)*MN0Q>^?daB`lpACkDgixZRdmSCkTh?M?!cR}e zq<_;V;lBpUp={3prlXyMy?yAP;Rdh1L+_lr$1PP@^v-zJ3BjPd;WZ$&+|I(^E_Z-S0|+R zocjubcU|Q{`Ef#d7sQ5yqfn^2aBNbw?4T|XqMz29FuTwC-pUw1dlGg1R(H6tMgKe*r(Zy2L?z6i_8Ma ztQVdeRqA_!qJN*Z9q}<+%teuqejlF^K3?mp)lR^g-?CH=&Lq86(;;MOluZ@VFPhyp zDDru0mXIUS>sV1m+h$qiT|ga9Sk%=yOkFWO`(f+qoVPU9Y%LEgvZL=b1PvBp`#oRo z7H+%`Ch$qxx-VQX&5t=SRDBTMozq|&tG(;NW4Odw{+{%D-12P1y;(8s^RRPv$xcHW zI$@Og9&AaFu6V!k1E})xqsw_dbJT2ohRmf!CKz2filX4C4RfO_sfe$QpbYQl)w#O4S?c!~3_-Od>N3?zQr#h!(6M0^c&dv!e&-OWGUG^*d(^g?zaEcjiu}}wBgP}l~v}_d_{?W z@nEWA`r(~s0G99RDJJsDD2BOvocGynK4x5a1{ZaeAnW&7TWc=7zE-g;Tw*R{==pg+ zAVt)??*Ad^JQPdk*KoE4OYxUf3$!mu&8A_hL$irj#>i(oeSaUC`0X8eaZmN#srqRE zKy25?uE*Kb3EQ+_|1NltwXZRvb$uHw)huc4=t*KnVyzp!+RZ{|QxIPd)gH3$=kW(n zrzkOr#1~k!@x{B;gzM;p7l$^^I3$0)VfN2rOme&Zev)H`}$xd8P;y~?l z2HE7@A2FX)SFL_dQ`Gcp%~$p=1@OAxfVr!Pk3*-5Hu`znvrfDDx@RzpAZV}lxsedD zv?Trg^{&ZkjZ!rY5^mad+6UyuAgSOsm^?#`)g|A+lMU|MaK_r15Nzqs?B4#sR}`b8 zR`IZMr^K`r@vYHSArk0IZ$2sPYvs)WN5slk;(vF|Ub(ogtj-85tR=rBTcC9!w%^~= zIYbnI@o`q2e`fRPvyQt*VnvXfAG@V6H|nQES}mQ$;`vPk##1caCYbvylnxdkz8%WQ z`#kieCO;Z2zIxa+4yl{-Ps&Gax<)NO*JBq28jkaqVpL_>0*`W1^Q{~H*^M`J9m@<# z98@BnTHt4x%6DM-M<(X|skfwrC{Npm#DqBnSy9js-xXDa^sDeQ@*E}7txi=MU>Of` z?i_nhJuD@tQ>kNzd|U<3l4C>r_wom@;~li|uC3pq)IQNSe!5eo$YtbfSC^c+FZLoN z07C*zRmpH1o#QkHRyY2452|#3CDK28B(o>w@q#_kxR8<@*reXeG>MuD{o#~>ZCgMY zi>G1vQR;o&L3KFHf$pN9VWOMTD>le_2m8=(F3bepMqU&EX%$bkE86lM&V^GD(*FTU z;m3pIDty21WC`C4*SdNyI<=j|JooKG1!NHuL#&Ru(L?J>M;Iz|KI!I?%MUjcux!-V z%M9(cAP*l2qAzhLIbkJE0`!GH>?GVU=d8TkWve!;9elcV4^hv z4x6}8pzO2;wVI3q-f#;_EsI!!MF9baMlV{(6fY3odrs|4{{ina^Zs<6_w#(u%ySM{ zV)jDBWW`&E%vAVeq4KAXj^r{UA>;E$F@%y{JQYc>!2HjgbC;1sdP>8P#rsA-rvCzAigod!V!yMkRkIe zb|p->l>It%Z7KI&-C%F0T8zUfgF;Q2wN4qw)Dj2!EP-EC+l-}<445%|72ALrA#4`3 z8Y;H{T<(3m?x$IvA^0s?QginuL$?uV8d2ZIX4N?fBhA3vv^@oHcuiG}9fvdWJph_EjncUN5zw`1M+IiLMxqRAYHM1` zXbu6fi+$ootTe3@^>LQt71@}9SjdW!%+J}B3IJFIw>b4gc^vC`n>ZXRBMN%F&z1y8 z(~~0I{HN^?XY6yN`_{>#-=!>qA8*de>xw>WC`8W7^lP!sOy9}*sFe$5h^v}-NwuB{ z9|=^Q3VB~u_FKehexAv%bp3*MCJ--2eG{u=%0g&-56n|Uu5p-NB>%Uyu~S@l_}9j6 zbVA2vr>tNW?O_gHZxpK+8MmuQG{I`taw;Aw7$*w6>+>cbCch}lcn!eI{lVH4%76I7 zd+*zd28vB^Qq;#=4*zf#ntlQkt3y2G8SSzlXKa1dQg;ku7sU5_#`8U0!{+~4>h8EV zwx+%Iw@ZF@CO?j4iaS$6B1(=7`)2nhc7yfrCMU*EDqA1SDYEt?r_D8eWo1tpPeOfj z;;~j4q`nAqT0gtuS}hp+d=icAyN$*F62tLu&T&!Qpm2rd@$MI*h!!$V zM7CRSO3YOlUHZHz6LRx%v%J$@+>l~Ny4f9g`_~4~#;Gy>FNdeoT+CWO@Qt{z=|t+k zt#&Gm9SbVo834Sd*xaoAb3dsBRz8wHwY>9Rf#*m~TVEtw7&(8Mes(1iAEG z<*>OPk80j2D$be46b#-`hMgWA;b?lkoEpp6T@IH4)G1P+iF( z%GA9uUakkyh_t2?#Rf&FpGzR{(;#&$9KE4v7u&(B&WqypCWGSOxnkVp0W$PVu*lX zjYIt+f+9H#QcK`2hcXlUu7UKD#X+Evw5%Y%;N^s64}m}*FOf*Fg#K3mEPQC7ZyXmF zOLk&MXk)-*QXKz)$k3ETB4JK5t^=UH7jCqd6CSk-Jv1NG|JpokHy+{|Qv7^<6W?3? j?bj1>ZFvw`c=x^f?}DQF+aX9V5<=>|W&a!cfBXLfs%7HJ literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\271\340\245\207.mp3" "b/platform/micro-services/search/test/mr/\340\244\271\340\245\207.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..c9879fb83f0ed07344fc2d777a714b44c82fa8dd GIT binary patch literal 2112 zcmcK0X;hPE76;#%ps0(5bGAuGO>_Su^C_$(U(4j1;?9aUK^vs!a`l0isbH6?3{_pR(=YnNg z2ZXn6T7`6JO%w-A$hi6sg|tK{uX94_OS-bbAhZhzo@=}C5b%SM4lQ%lWkUQdofnUImq&;4Lu>OfhD@>xwSY#M-lb)3DP8p=r*a`4~1 zZ@MU8?0UJaO^8YG$L_74G!EZtJDc-uwll z5srEX)JWa>Wqow}_6_$a&!9KB(|;Sw{EVPgPEf@v@=W%m%#Qjk2I=mBpM2@7t{2$vNTd_ zd99Cc%bF7(Tk&klB8UXU*COl*OGNcYLe*<)I{U0aC;;;_oT%X<=Qfnx=b2%N>nFSR z-!@Ytd8sarTDp-_tv~K)3+{O>8NtiHXoXlLGiaApBzuel09Zds*bG0q==Z>Vb?&1~ zCnDl&5=QE>puzJZTBC;Y97>rs!Y2ai;8XgU)%Ax3#dR;bkNB(Pr!Ylb;=D`54gyh; zK{aa78h<3o@T9xzf*g$ObqqB5&36#RoV*ouYLV8y16HJ z`|vj}?^m0QaD${jhmT2_1t#~RM@1D|={Lv2>RVgD=GemMb`%giB73C5di4nZUSR)z z0N&s~F*;c1O|%LPb4UIA{@Pu&62v#?c6nJ|Yu+-zsiBIyz*1XDGOL--S6ifHM7=o_ z7iTMt4$SS81$~eNRD;oYupBrR`Ys`1rlueEmzU~Mi}=*qImranawnu$VXVlWz%7l2w6fg7TRaQ_gbE29;j^Fc}r>55RVv@^dx^&NWt1cxz(c+xG<8`3H<=RKzHnq zv}#8SmAl}jBG}K!x1^+$r^XA`)su(uBce~2uZO}RxPpF>NA1jy7YdE`pb`I1I2>IJ zJ;Sq!$Hh%~h6foI_uVHV1PkpPI4mtFuGEwMbLoeG16UKk`A=0R(7JjnhB2b#M4^*$ zZ{*ge@RhzbyF3Lk%m~9J!|L{VNxA~ncXBMnU@t|4_%^s2dLGoP&7M5Q8HdU3adZBD zSH{*XW8+It6&0?H+*@U`;YFXyYTRuX(#39i=kp3U1|CdX9lM}wpidzhWw^ zd{3OisTfFxWrvfUm8?wmDc+&~reUj(p~d71N}5~XF~44;$l=A-`VkjZM;~pK^Z}Mu zu^j>bzNHavZrZ*g+}Kg+;Vk8RbEtN}cY4cD>8+Ixi~eU!vKaA$aD%6qP){a1I^D2Y zyOR)EL%9mif#WRp%9jf%qt(H<@I?JoWkVDaMEnSx(2@qVd$G4pUr#a0V$8luLEAO_)XlpqdHC4- zgJhNiuE`G`j-xdCQDzE!Jsdopj}plwlJD=nB1G a+7*K95A}cj4%uJwAOANS{1MOpxBoAPN$u_c literal 0 HcmV?d00001 diff --git "a/platform/micro-services/search/test/mr/\340\244\271\340\245\213\340\244\244\340\245\207.mp3" "b/platform/micro-services/search/test/mr/\340\244\271\340\245\213\340\244\244\340\245\207.mp3" new file mode 100644 index 0000000000000000000000000000000000000000..e727aa7bef95d21682b4392b0a003d970deed55e GIT binary patch literal 2784 zcmcK4X;f3!769OzfJzu71A`(c2qKfrAX*9`j3QBt0c0vO2Fj2q2x1k&5C)?VNO%;G zGD#r}RfLL2Wr`IlAhQTE6%d3LsRGIn&bzVet@oqvN8eiS?Voq={hfX8I_H27Ry+tx zNL_(MtRBTdHhN5vpG_d~IEE>N1RoAfAwk0S&i4y8J)8O+1oU!ZnB^0j{Mi80MCBV% zN|Uo8^OtZZW*v3)!dK)EuB^u?=OFU%q3OxJ#itu&Ft#FA!ZoEAe#|$iNhroG3Rs)-o#FPMQ1*c7B4*Ps5TLwi?!3lxP*{Z zzsm&OP7_%E$7%_jN?dWc(YM*WXFQyuz?YF(I!r<2{2mM_rgn|*z3L00)x z^vH(d?0_=)YZwF{-Q`V(oIggMe_KW$WZs1r0(@dcL*P^!f4EtV){>!Sv$<&4&lC82 zrPXUvpoueV$K4&74C3{Un=YxObhbadY%mD}4d;oMSjMAGE)8&NG^WkO>@VYa{*$92{)p*c}DGgB3_OFkma z@UZdz1Pcx#Qt28+R)ks_Hv-Am=C?G499uZ<1tS}Lk3%sEFDq;3&?wBfIBpNJC1$G` z0NdF*E_gQs<)R!2O?zh!N6Q!rd`an`#tdj;FMErS<%Udu>56ZvAJy)wqcLT55b#XJESCWpI1F+-e1cqM4qa7h* zLw3Qv&+|(mYV)Dj_Uzr&!v51TH_MJd@u-oitKCnj9)vd+9jt|YmHW9pnD)-6nrGn! z0?Qkwl|oXzcJ4Cu+1@Pru3v9CN2MzK_-Rcb6L zNmeAbooBs-&;S7S?WL`+|B!9hE0!lUq{R04b~n zv;@e?M$t{xc`X6$RyVQxQkSF!z7}TWD|x7=Ky4x1d26}Y-8MbgU3bgH*}2#+H0IFO z=)}P0ADpjU&l_ft&*|Bx+iiHDT~I-RYUh2ATRvm-23)Bt@g&7`7UkJb&gBPk($b6` z|6&w%a2NTGWr4syjxKbQLY&5S*Jj`E%Q@_2q8eJu&eLa) z3Vd6oYZ@YmbPDuoo@hNljmp{2j(o64B6=FtUk}MEFn0as36;k5RuT4n9P3mkvs15_ z(D>fl3E6AJ>hS9=JW7w)FND?+5_z+A`sz`GA1tJV^vsTJ>*dlP&Q}P0DOq=!6hw{* zVs;j#Bxgu@lTXgoxcJyV`IZW9ALhvoO6R3TijL~Zt3fIMqAuT3(n=n_ z5ULNW>R*5do7WFmiUjPzM_bV>(7_1rdu}w+uZp#X8bU2vlrlMFtKK>}9Bc(M%-R{! z@M!{L@_G>(0C3}ucXluzfA+DglcmsL0ktUV`|1{r8Sf`_wXff;HphT%uyYykOzua# z4ipPEN}{trYIoiI@$d-sp3(2(CWG9i7wzWT)Nf+q1n}BG@FL**JL;L0tIv1s&esy* zExQVQ435>754{>?t0JEp4Dr9b5!LhIc4?Lha*^`j5?aE0IJ)Ub+&Sd3{yEju21f}d zuMN)%2f)Q5WRciwE}%C_@-e|+`1sF^G0iWRvVB=a@pwZUx%ia`fiI0^8N>br_3GC3 zpPGR=+%WGoU{or@o60QIfs!&VE?Rf$*fPWj1BuoRiujvgz1p$G1ug^kpT}^)IB)FK zMF~wM9VH3Kq)AQ7R7qv@I{cn)cfQRO_zIXJLJmAX)xV)750~7kf}3s&j0h6LOq@Uw zJcvmj7%7Uez|U6F^?WRB?yPEa!35hOCmTu4eU+l5KwI=a+tWVXBuD~VqCrowcY%b8 zzwNXxs$Af!W2S4_(3l&WnwR?Na{Il}Z(e7ac{+)j1HMa_&-+QZYx_Zer|8bbXad&M zU6O;yTxg>}Luw($Qgp&`x$j+m5Xk)I6O}%~wzT^x?<5g9o5yhR7j*Fg-w=yB$%Mw7 zspXDeSoUwu?*&UVn)gLP&mU_}!-kUO(qFO%9jB`ET^;0wRZGRwy9=vC2>p^H^Um-_pUGDVd^5}+ZHdsULAH-!LM+_YHS9BI>N9L& zXH(zA-R+wq-@_y1&aKC2k!3pIG+6U^n;L1YcTAyV#O78qHXzxu;j(R-?bD2P>+j%g z$dI2NxC58%wAr8fe1A`2&oxFveQs<{PJoz{95RfL*M7f|-P=MCFS1Hi>ef`4GG!uW zB_`n};CK77V~7}I*-Q*yQS5y~`~U{yN17o$joorqy&Ve|Y$jH_?>m*8MwhVne(?GI zCt+!36lm-WRVJY_y89n$_To0X$z7Zk9F{E2y5A{lWPNbR9PaG6V_PP5iKMiWwm)7} zw5{*%!Ej)Y!oV3Y)ytOFiZ*b&wu|H8Wv{Gt@8Qv}xcD5Bz2N`2V}mqIp#d^ggSo2$ zSJAw_6TWjd(sxOmtUqYymw**LSO8WAQ`HB5)%oHRr(0{qsn=zt#x^C>*};NnZJ@2E ztIPds{FxK@{%D$pFcgzW{m+GeZT(wc{_RIXkdPe&$N$v-a|prwRsZt;)?EKx(f`~3 ECnd$YRR910 literal 0 HcmV?d00001 diff --git a/platform/micro-services/search/test/test-dataset.csv b/platform/micro-services/search/test/test-dataset.csv new file mode 100644 index 00000000..7a717263 --- /dev/null +++ b/platform/micro-services/search/test/test-dataset.csv @@ -0,0 +1,16 @@ +आहे,mr/आहे.mp3 +आणि,mr/आणि.mp3 +या,mr/या.mp3 +व,mr/व.mp3 +हे,mr/हे.mp3 +एक,mr/एक.mp3 +हा,mr/हा.mp3 +नाही,mr/नाही.mp3 +ते,mr/ते.mp3 +आहेत,mr/आहेत.mp3 +होते,mr/होते.mp3 +तर,mr/तर.mp3 +the,mr/the.mp3 +ही,mr/ही.mp3 +पण,mr/पण.mp3 +असे,mr/असे.mp3 diff --git a/platform/micro-services/task-manager/README.md b/platform/micro-services/task-manager/README.md new file mode 100644 index 00000000..ac691172 --- /dev/null +++ b/platform/micro-services/task-manager/README.md @@ -0,0 +1,12 @@ + +# Task Manager + + + +# Supported platforms + +* Local unix environment +* Google cloud + + + diff --git a/platform/micro-services/task-manager/source/task_manager/__init__.py b/platform/micro-services/task-manager/source/task_manager/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/platform/micro-services/task-manager/source/task_manager/engine/engine_factory.py b/platform/micro-services/task-manager/source/task_manager/engine/engine_factory.py new file mode 100644 index 00000000..133f4e6d --- /dev/null +++ b/platform/micro-services/task-manager/source/task_manager/engine/engine_factory.py @@ -0,0 +1,15 @@ + +from task_manager.engine.local_engine import LocalEngine + +class EngineFactory: + def __init__(self, config): + self.config = config + + def create(self): + if self.config["task_manager"]["engine"] == "LocalEngine": + return LocalEngine(self.config) + + assert False, "Could not find engine " + self.config["task_manager"]["engine"] + + + diff --git a/platform/micro-services/task-manager/source/task_manager/engine/local_engine.py b/platform/micro-services/task-manager/source/task_manager/engine/local_engine.py new file mode 100644 index 00000000..86bb3a49 --- /dev/null +++ b/platform/micro-services/task-manager/source/task_manager/engine/local_engine.py @@ -0,0 +1,11 @@ + + +class LocalEngine: + def __init__(self, config): + self.config = config + + def run(self, function, *argv): + return function(*argv) + + + diff --git a/platform/micro-services/task-manager/source/task_manager/pipelines/__init__.py b/platform/micro-services/task-manager/source/task_manager/pipelines/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/platform/micro-services/task-manager/source/task_manager/pipelines/make_dataset.py b/platform/micro-services/task-manager/source/task_manager/pipelines/make_dataset.py new file mode 100644 index 00000000..cd9b09d2 --- /dev/null +++ b/platform/micro-services/task-manager/source/task_manager/pipelines/make_dataset.py @@ -0,0 +1,32 @@ + +from task_manager.engine.engine_factory import EngineFactory + +from search.api import search_for_data_book +from ingestion.api import ingest_search_hits +from forced_alignment.api import align_dataset +from quality.api import clean_dataset + +def make_dataset(config, data_book): + engine = EngineFactory(config).create() + + # search + hits = engine.run(search_for_data_book, config, data_book) + + # ingest + long_dataset = engine.run(ingest_search_hits, config, hits) + + # align + aligned_dataset = engine.run(align_dataset, config, long_dataset) + + # clean + cleaned_dataset = engine.run(clean_dataset, config, aligned_dataset) + + return cleaned_dataset + +def run_search_task(engine, config, data_book): + return engine.run(search_for_data_book, config, data_book) + + + + + diff --git a/platform/micro-services/website/README.md b/platform/micro-services/website/README.md new file mode 100644 index 00000000..5699818c --- /dev/null +++ b/platform/micro-services/website/README.md @@ -0,0 +1,10 @@ + +# Website + +Hosting datasets. + +# Usage + +peoples-speech website add dataset.tar.gz + + diff --git a/platform/micro-services/website/requirements.txt b/platform/micro-services/website/requirements.txt new file mode 100644 index 00000000..66ad058b --- /dev/null +++ b/platform/micro-services/website/requirements.txt @@ -0,0 +1,3 @@ +flask +flask_cors +google-cloud-storage diff --git a/platform/micro-services/website/source/config/default.json b/platform/micro-services/website/source/config/default.json new file mode 100644 index 00000000..415f1cce --- /dev/null +++ b/platform/micro-services/website/source/config/default.json @@ -0,0 +1,10 @@ +{ + "peoples_speech": + { + "train_path": "gs://the-peoples-speech-west-europe/peoples-speech-v0.7/train.tar.gz", + "dev_path": "gs://the-peoples-speech-west-europe/peoples-speech-v0.7/development.tar.gz", + "test_path": "gs://the-peoples-speech-west-europe/peoples-speech-v0.7/test.tar.gz" + } +} + + diff --git a/platform/micro-services/website/source/flask/app.py b/platform/micro-services/website/source/flask/app.py new file mode 100644 index 00000000..f4b2e006 --- /dev/null +++ b/platform/micro-services/website/source/flask/app.py @@ -0,0 +1,22 @@ +from flask import Flask +from flask_cors import CORS #comment this on deployment + +from gcloud_sign_url import gcloud_sign_url + +app = Flask(__name__) +CORS(app) #comment this on deployment + +@app.route('/peoples_speech/train_url') +def get_train(): + config = {} + return {'url' : gcloud_sign_url(config, "gs://the-peoples-speech-west-europe/peoples-speech-v0.7/train.tar.gz")} + +@app.route('/peoples_speech/dev_url') +def get_dev(): + config = {} + return {'url' : gcloud_sign_url(config, "gs://the-peoples-speech-west-europe/peoples-speech-v0.7/development.tar.gz")} + +@app.route('/peoples_speech/test_url') +def get_test(): + config = {} + return {'url' : gcloud_sign_url(config, "gs://the-peoples-speech-west-europe/peoples-speech-v0.7/test.tar.gz")} diff --git a/platform/micro-services/website/source/flask/gcloud_sign_url.py b/platform/micro-services/website/source/flask/gcloud_sign_url.py new file mode 100644 index 00000000..03834add --- /dev/null +++ b/platform/micro-services/website/source/flask/gcloud_sign_url.py @@ -0,0 +1,59 @@ +import datetime +import os + +from google.cloud import storage + +def gcloud_sign_url(config, gcs_path): + bucket_name, blob_name = get_bucket_and_prefix(gcs_path) + return generate_download_signed_url_v4(bucket_name, blob_name) + +def get_bucket_and_prefix(path): + parts = split_all(path[5:]) + + assert len(parts) > 1, str(parts) + + return parts[0], os.path.join(*parts[1:]) + +def split_all(path): + allparts = [] + while True: + parts = os.path.split(path) + if parts[0] == path: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path = parts[0] + allparts.insert(0, parts[1]) + return allparts + +def generate_download_signed_url_v4(bucket_name, blob_name): + """Generates a v4 signed URL for downloading a blob. + + Note that this method requires a service account key file. You can not use + this if you are using Application Default Credentials from Google Compute + Engine or from the Google Cloud SDK. + """ + # bucket_name = 'your-bucket-name' + # blob_name = 'your-object-name' + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + url = blob.generate_signed_url( + version="v4", + # This URL is valid for 6 days + expiration=datetime.timedelta(minutes=8640), + # Allow GET requests using this URL. + method="GET", + ) + + print("Generated GET signed URL:") + print(url) + print("You can use this URL with any user agent, for example:") + print("curl '{}'".format(url)) + return url + diff --git a/platform/micro-services/website/source/react/App.css b/platform/micro-services/website/source/react/App.css new file mode 100644 index 00000000..5f0734cb --- /dev/null +++ b/platform/micro-services/website/source/react/App.css @@ -0,0 +1,38 @@ +.App { + text-align: center; +} + +.App-logo { + height: 40vmin; + pointer-events: none; +} + +@media (prefers-reduced-motion: no-preference) { + .App-logo { + /*animation: App-logo-spin infinite 20s linear; */ + } +} + +.App-header { + /*background-color: #282c34*/ + min-height: 100vh; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + font-size: calc(10px + 2vmin); + color: black; +} + +.App-link { + color: #61dafb; +} + +@keyframes App-logo-spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} diff --git a/platform/micro-services/website/source/react/App.js b/platform/micro-services/website/source/react/App.js new file mode 100644 index 00000000..3febc05c --- /dev/null +++ b/platform/micro-services/website/source/react/App.js @@ -0,0 +1,76 @@ +import logo from './logo.png'; +import './App.css'; +//import getGoogleCloudStorageUrl from './getSignUrl.js'; +//import getTrainingSetPath from './getTrainingSetPath.js'; + +import React, { Component } from 'react'; + +class App extends Component { + + state = { + train_dataset_url: [], + dev_dataset_url: [], + test_dataset_url: [] + } + + render() { + return ( +
+
+ logo +

Download the People's Speech Dataset

+ + Train + + + Development + + + Test + +
+
+ ); + } + + componentDidMount() { + fetch('http://localhost:5000/peoples_speech/train_url') + .then(res => res.json()) + .then((data) => { + console.log("Got response: ", data); + this.setState({ train_dataset_url : data["url"] }) + }) + .catch(console.log) + fetch('http://localhost:5000/peoples_speech/dev_url') + .then(res => res.json()) + .then((data) => { + console.log("Got response: ", data); + this.setState({ dev_dataset_url : data["url"] }) + }) + .catch(console.log) + fetch('http://localhost:5000/peoples_speech/test_url') + .then(res => res.json()) + .then((data) => { + console.log("Got response: ", data); + this.setState({ test_dataset_url : data["url"] }) + }) + .catch(console.log) + } +} + +export default App; diff --git a/platform/micro-services/website/source/react/App.test.js b/platform/micro-services/website/source/react/App.test.js new file mode 100644 index 00000000..1f03afee --- /dev/null +++ b/platform/micro-services/website/source/react/App.test.js @@ -0,0 +1,8 @@ +import { render, screen } from '@testing-library/react'; +import App from './App'; + +test('renders learn react link', () => { + render(); + const linkElement = screen.getByText(/learn react/i); + expect(linkElement).toBeInTheDocument(); +}); diff --git a/platform/micro-services/website/source/react/getSignUrl.js b/platform/micro-services/website/source/react/getSignUrl.js new file mode 100644 index 00000000..14ba4aab --- /dev/null +++ b/platform/micro-services/website/source/react/getSignUrl.js @@ -0,0 +1,5 @@ + +export default function getGoogleCloudStorageUrl(path) { + return null; //file.getSignedUrl(config); +} + diff --git a/platform/micro-services/website/source/react/getTrainingSetPath.js b/platform/micro-services/website/source/react/getTrainingSetPath.js new file mode 100644 index 00000000..372fdbfb --- /dev/null +++ b/platform/micro-services/website/source/react/getTrainingSetPath.js @@ -0,0 +1,9 @@ + +import config from "./config/default.json" + +// Get document, or throw exception on error +console.log(config); + +export default function getTrainingSetPath() { + return config["peoples_speech"]["train_path"] +} diff --git a/platform/micro-services/website/source/react/index.css b/platform/micro-services/website/source/react/index.css new file mode 100644 index 00000000..ec2585e8 --- /dev/null +++ b/platform/micro-services/website/source/react/index.css @@ -0,0 +1,13 @@ +body { + margin: 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', + 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', + sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +code { + font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', + monospace; +} diff --git a/platform/micro-services/website/source/react/index.js b/platform/micro-services/website/source/react/index.js new file mode 100644 index 00000000..ef2edf8e --- /dev/null +++ b/platform/micro-services/website/source/react/index.js @@ -0,0 +1,17 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; +import './index.css'; +import App from './App'; +import reportWebVitals from './reportWebVitals'; + +ReactDOM.render( + + + , + document.getElementById('root') +); + +// If you want to start measuring performance in your app, pass a function +// to log results (for example: reportWebVitals(console.log)) +// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals +reportWebVitals(); diff --git a/platform/micro-services/website/source/react/logo.png b/platform/micro-services/website/source/react/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..3014033a2addaafa90bbd8debac2f36ce7d6bcff GIT binary patch literal 23989 zcmeEug;!Kx^zQ&7rIay*G}1LFpy1FcARr*p$bg_jcMT;7!jK{R<*c85Ty=m~#B!*hXW5C}U&Ls`MV=l%9voBMUOi+_JMbncV2 zs^rpUX3_0LXz0?NeI`ROB^hOE95pgSRDB9Dg1vD|95!t zHxf#$Z;duqPh5GK$h{Gy@BEhgzRN$%IO|>^VQ0iHA<9<2BWlnvO^rmM-hs@^0|#On ziEtdEZ0;8eK@@Um!aUHi=;R6>jV0pK8#Dvh2O5bx+!~T)7~W4;Hy%3DMbWc%>=SiZYg*5jI9A2e${ z4Kg_3x#S7Hhs4kg=UTi94ov0laqIT{)tO3aOk?@ciba7S_81M6e( ziRTfTi+?!sl_aK?wwsu`>TSz*9A#=QL#fF(!P9)Vzsw#Xpr*TK9s&}nwfxo(sKY#p z#&XnW*%j)){6RUD&29aP2M^LxAkb8+%);I`1(vqIStD?@@3p^Vc zEwnmoU*F*jjVr@2H)}8L2kFZZ>AsLj4&Lo{73%+sk#Xy&xk+&Wo@F4CBEKyW)4B!A zk%>dY$i&HCtJ4nWMwywvmpD?J8)Y%Z9k_H>z{NU$P#zS+(8$-2JUgIhivg~y6C3-3i#%#4<()837f9-`jO&R8HKA0Tco)Hd#;$@51Ca+|pnaMvaHCcyw*oz^Vqq-`0 z{_usb)CvTyK9`MWM%<(-udNaRBEZfmL6-CSIFd2Luabx5RU+rhAcI@qiZPPC0&5aD z|9=^nWA0*$xBuQ@ym)650R498)JyFxKEEfnncP<*5AT#-iz$dD`+i3@X6XM5>~R<{EmEyv@N>zm`f&PvhPRE&FHGoOc(V zfpe+#xbUW4Dfvbe?Jz8?WMgK4i$z6;TNmr!tf@K#Mp&Zt@3w7h;tcaRe0XdvWLlSr zW>5W1)s4ZheLDt*6;kgau|LwD+y)2?)s^TS!nCmS2@iMISg>72lVAe!K3dQSDe$@~ zW#?RlPxJb#;*1jKvblJEC6UtX+InWAXKx9nk2Y>nZ8(>lg9T+$;!}D6BOSS0+UIqX zhWWMU_y>Xlxu&A7mR(ZP80WPzX&%rWc4QpB`ri^EJm~mX3kHOldW-&_r+GAm^yB7w zu>*o%rHKggBdfRj=u#&6#=53qhq{zsfY>{uuZE5(pD(29R?U8A|KytIVue7gdU-}t zRD9T6W4n47s^dkiV0+0HhFe44eh;W6@)OUiV9%AtbJ@N(syzTA_Pt|102zM?TqrjqNKPvWcvC$?iPsQjESF znVxv3kjCdvlx?0ZE_qYAl*v7n7Y6$SA0HcB1xt-ghVlHiI+9|%7fM{J5f{lU;jNz+ z1LLLlLtQ#l-d#(B{Vtnr!=hL}jpyCL5cD)+k8^m<<)QpUcswfw;zx;=l5Y}sPrtkx zN@e$-^0)^6@r_~L23CSf&AWzdOTMTV*YY-cMA_~*Fxe`Uq%T#_s`=$Ut{qUe`!Sx&N5Dq);m)xE7H7Y zTFV6Pfz1zwxx8|q;q+sbO{xESO|em2yaiS9>u#Dp#gU9>VPPL4f9%&*xwCiP37JT# zDKf(WrYDie`t8XLN6{p(BrEIL;AGhIzN>C3dp~eZLP(W+r5Z;0D)Qd=OlJ?XJFR@r zT6S-=9dK(r*>akYLxMyUp4)#Z$gsx26PPp+gc<(Diwb0hURzRxx zy??XepUREeW_QXkyOnb9@R_So4imKxwxFvYl+u*O^MG7hIpoL5R98Is97H#GWZl?A z+{qwybY>jGbw4HHKqP#Yvgi7-%KhXWCMcDeuFi9G(SXsJ=@tDsBECI8#}4zoCIPj? z?2`nGq5%K%jhj1^1`W6s!_&Z)^iM44RjLV;g=D(u*aq9X+1*Oa6I3o`VH3%@UrVjHsKlL{D<3y@Z${7u&JM7C zVL>IxXI%)@r2cNq_QmdQmk+>Zt}qZG&|CToMu%FnxE0PjCZ1gRG(`&XK_Xl2{^w$h zh+1T+psd+jswOG0-&R72W&ow9~vqVrl_I% zuz2cAM}CgB8ee@aoA|ahpH>+DrRQ_Q_I&cC`27ifGu%()S*E}b`LO(jHuzqoG!dLL z>@Z~X{ptQ^ryED{FE1ZyKC9qd1e&oT`7A>q%%fe7Vl9J6wSUVkZ;Lt1^R6o(^{Fa* zFMdl?)XS5jl*CoM(}8;Qb+Zc;O0oWjJ)g#^(_|;9L}Etb)hS*tD6RLOX%ANA+7*h8 zU^($JR8v>Nfw9@5u`Sbs=U|VIZsRLA8d2f0n%$tT zt@^bKm7k#QoX902$+~>rKKTYmZ;jD@OeXbIIf{+dPoh-^#oz zsqGss=BHP&#f2q!bC2}y!@=Z`+2 zOyltMjQXNybV?!5lMvi3mdtmEPm9A$F{PNk;WS})eQ60pt_>|=QXctZDY-Z7IlPa* zAIQftUh@V_bTnQnfamZIH!Qpj^@4CN)jmmXJ(4hreyG|ub1#XpBFG)5CLhPfwE<77 zRFye;G4$j)Wp-clrJa(Wea}=@Px)#sH5oL>Ox}JrwTsq>VSQ}il%9Xj?8+UvC^LsQ zjZLy$;V$fMRRJCfms_GvU-+l}W7VZ~3Lq!Yv#UR-NF7J34@GC2%+H#yMq5AR=U}nl zJ(`a4*Ey#@DdsBDL4%F)XRK4vB@3prYV@bc7xy!iUx+I!dUd_UUg9=yaxityeMW@^ z#xKSGJf{D zBsNFbk%Nf$OBUZz7qzy}fqhX7JKTF>cRhLr?P+5?-gdX8UDwcqT0URgaJGwn@S3lP z%>Jy{<0`Vof!)J5^oRJu@Ta6yt@e)}j)f5xMK-jRhN9*(3|~lp}!@Mt*fA z0s#PYIWI~rvar|pUHZE%eZ3n3c`moaX4EM=`uE3w#R}n6 zRHymQP%Vw1oa0j+(y#J9Ws?Z>>@zXNT#~<`D6PA5G~HWT>!ihM8Pr%1xVbkp^(Rya zW3t(x+U{O=ckwvJbhB+HzsO`P{IJru9>_>?Xs~z{o8M3(mlS?=C$)+=og<&j8ePm} z%w#K8azFjzUZLmTFpr$~*9|fr59-0w13kQ&`7lJU%O3}B#x4VbFO`0UfvJ3K+H$w` zB!e6(U&bWo@43wtNa8opw`*Zp)Fn-1?D!8x+Jd)`bkL{~$jC5GeP^)z2j69Ba_G;v zpRu@nk>J;k-)C_P)nJ$2=2as+P0_u4es}LQK4|a-XXR;~Kk0`x)gLi1fAOuurq>v; z-CoBFvCi%8bs5#zcvjUh=&y^XAIrPPTGlA3N#*4VeV#qiu6GY#1bp?r9)8e#w-noR zJwP~_*UOb(MV-v=x9U&x&&QLG7QAt{pJnbSsG;L`)3gx9r?jys?a9JW1cY{S3-d0V z;(D~8+bZDd;!Gt=CQC(cdIw&KP4num%%3_iPhqHMp=4B+n`sqnm1X@6>^Z-c%+BtC zGAjv}vO3OrymdW~dSiT=#8`&JYPRr!fYj5E>+9QpOVC+C&|mg@d0@G(IMl{$Ef4uE z%pMNqlKEMoU0*$sBX#;Xhx*5DDvPio-?%VD5j`rXdO3EGzep-Zt-rfXvp;)#wo}~C zqO8O%`{(MM#e*FH!xV4jn2Ry1_@w*9H_lp}9<=)W5|G0Ti`0q#e;ztzh z))f;^e)GyQZ4@@+OzPDXcjnc9;%^v#lKs9uFiIsdm+;jWLL5(bXe`4Lz86e)!2dhb5wHq)!=o>$qvTbl8N0^|wc|`DX(>mWkR`Q0#aW?`cyi*#UbTic zma36Kry$sTmzvj0Ju_}Xk+(SJt~IzBO7^r{&>%+{HG(4IseL$!P&!f8 zFI^WHzI7aS(+!_LE0GB2NeuF*_9zr+%$eb{HRUd+)S zl=tx-zFJmM$042=ddJ-8PvO;F^@>AH5h*|S-<+9gaCZTR$p_yl>_tL>=*5b*Q&Frd zIbg=Wmz=RHYI)=GUd}}{>UltdWvi&ow~Dl~cf}0v6phfyNwMEcizxxt4uv= zQEs5~Nf@}k#(vD<+A8mA9Z`i!!D9;#T2UaZ%;7N}fv-BkV6=*U-rl{tAxG;F)hMd? z3u3mJi@%^oUCX&lvF`6+8p%OBEE7*mZ`)jEbO~ZhF2fr76YTQ?)AF$; z_q&END*3y&?E&6&^UL5-^j$&i5$_K3QbjT7zjrV|f9 zeq*=YA>)$Al{fM8{Q(_iALr>HVov%_%hd2W8jhA$y!b)Rr()5u)YAj9=fK&Cv+ufE zYZZN+qlH;@m`H0ZNd}*p-Q>{JDc$uv09~ka-sHPEz*b~~p1L)aoa*r9CcmPeQ~^ik z2UiHtRr6Q#*84rFHy%IO^$12xaCV}yLI_b;Q?qzgHL8PQ8F7=|PgNaT=pgf&-y1}h z!KFP}n~ff99GOEgzXKc%rN(mn9)|npq+U9-&CSkX_$BX#<~Gi14gmP{_^fWq=U zgi=F(;nUMr>6<@hqi8wcR|=De7~pBg@Pg0oe&-v>acPgy`E_esv)aO+=?6ouAkd~> zHSmz&z-OJ}=6uTP&T)@!LbN^|V7D49jy&Y_SCBK(U<37l`0z-6)l!eULKy$iO(bA+ zN0RivjI7-*qj&on($X@!&6$I{r|&aJ_X9ZB;)-|MBMNU0UQ_e~9uCn{r14v7D|(*0 zt8<2)c5Bou0Xru$?-n1*lZC;a`+q>zq!Ha>RP^(2o+4Vz+@W|+W|_CHtEify!@K9V zH&1#&xlY65CsMAS`#?D~e%``3_sus3YF0=95G3UGv$fd?`<0<7SSGi5XJE*N3lO2( zKB*57KRUxm{Gzbgkw6V0fLh^aI^d~j_r^JtG-T^s!8rdFG+bcuXp2O!~#Rj}7A6(ix!3-ukRR<>Rf zYqKh;=LzsK_rgX*FLw_14^MpI?y@n>)I;jss9;!ny&zVbyV2jlVxclC7<-83J zwV|n!>Q}*-@WWJR;aflyi%tg+=-)L|0LurR&!9EdTQLEVV!N$BH@{3z28UBbUMB z0ubPzR#-^kY=h4SnR#`xIqdoWeoIiw#VMD6qk@Sg0vc{YNPv`|F(uNH~l=}Dt>sC&Q&!<~x%K)?Bf zrjq27MPi6-&8;4$g@abr3$Tfv;-O#tlr~oJPik#u9{XMvkx3c~vf5NM@U&!kA(zdM z?WF_hk1xu1cTij|mGuuqb|t+r3w>+v1jl}x+3-vFqVXEq6v~RCAxUpGFTSGsRm`~n* zDlvh_Wl^;gLNl(-fz!ZkCJVsPN?z%`*nzl{t`+Ij|Zq z@7uP>O3~WUjofs9I*}SlsqXu49s9Fr{!HGWPWw$A&~B$*@e--H`)$4s4jolH3{VXR zQn(&odNN)^?Q{gzqM%v>*mop3?AUtQ9p72@MZ^INEi5=)ytuS9ynA|!%c|Wn;OB69 zuga**CC?Th=E3voe1S!aXt`!^*4K|-qYEeMDi#@%Z{rK( ze8`UmN{NEz7Ec!9cE@cGwbO-S_)QxUtJ%)1Cd>EFTwx1H1VP-NA@?T zuO25j0e(_ex)bz+bB8-Dq!DyfH`ZRZ=RA@p>pi*ktNaulQ($kFfbV1oZ3KuRq@A5x zC}<1;yDvcmSL&0t>z}E=IT^|3Jyt(=nd7N*6Oa z=xE>Z8#SH(hHq%;1**hkpTTDq6|lPNo`V`hHlUWrDNHL3r6*dwKrG{=6B=5f=PBiP zQEUdXcKvoPg~AtoQLyL6eP6VOfO-UA61-e)&Mn050tBU7w;XCxJJ-LSydEyzv7N09 z{CF^U8QS3nVceOo2OvUiv-2G3N5QMu&hUe)6z7o54J6!8JVau{cK{7@@*)~pAR}Z9 zi?}%a@xYf4-PYN;@UL6BM`Jtvw|42pj-eO%M!^S@YEgWOC?Sd~xpbs*!rwUQA(Gn; z?_xQYB{SK$laDvUvr>hg%A+w}W;)#rINQI`p-L1Rm6+O*yY~=3EL0-Ag=DiUYm0vR zGEI=JRH2YN2IEh2UxNc>R4{TUn^~e>;9uK=mBT+j{@f=cVT(51+P(c0L={arUJG5Q zn8=LHK+M(%0TpR}4G92o59!IR+e&fCU`o9$3$yoo0s|invxzXUUw(XdH>OMJeAL90 zX_R-6w{$-~XI>O|BUWDlE+X_?6@T-k*3eus-B$&5KDGeq&t^iCu4@A$;lzrkwyn}( zee|Pb=9a|;icMNQ6#*4<{`&$68@SzSug@SP6uC6my~X|w9V~l9bXDp@psr`^A{1CE z9@Wqjw?5BmoF#x>UaY<#K=x#ws%Xqh(??E z^i_IbxeuM=j#ayfFR+*M>1#?d4?j8s?Tctx^s9l7F^^X-hB6*N2QZlV=v+MYC z0Gm1Xq8#rCvyvR#Z`2;DpDL#v#eci$01@26-{yuolRo|;@GV946l8(!=H>W}i3Sge_jSSU6>Ap&U{j7h+E$19v_7Z3cHdlC{5g=Vdv z{B~`x9=7mSL|L%mTF-X?T9vmDaRx4*=w)|D{~k#`nI)C?y_F-?nd&>RfJ87b%EsF| ztc9@gUfuj5LLLc}4A*jDZJUMZR8pphqBG-iYiXN1$CJYLxYzI1peq+elUJ8xg>LVK zAGK;!?G*m=Z>%VZ=8FTs)hh9Ct8iK+*Ss~Lgaw*Dugs`0FS6>W?u5wtu=JF{P%^K- zy7KNnxam*1Df9Rl8xim!tdwm|PqNT1M?K+_a%M$Gw@my~DAqQF!i<~4n*oI4h!Zo{ zlgZW($twHwe`*2d>{=vuImdPA#quiVw+plSu7s3OY39~<$uKOd&=e^T)ziZ-AIXS+ zc%(}cOjr+x;S7%Wkz?7n@aeY<-~1HrRmc!;$Rug`-aS9rVXkYw&h=Jrt}&YwDi#Dz z#XG7ry(hsbE*7}2kNJ#NK6I_8&|A5nZpLU@KYa_5l6_}sIjr{)JN3t_DX1$!dM&Q{uaw+gDxQwK!+nF_%iBVaun%X0^^`nWyvFc?9&BG}^8=tFHka<@BdHPmhV1GbnHyZwxO1jQ&4-hnTOuO&Vd3@O-Tu((3B63pZ4>1&i4a0 zP3JjoA%2*~VNWVbG4DLZtk-EwklGhmGz@igm3mHv2)UE*^^^Qff(In>{cv!WO^JL{ z!jZN%a@=+_-~^~RX2-wU0+!wzdnfPrFT6gyo5!I@J%$QWr?h>stpt@2TgyLN^*f-t zqclzy9uI=Z3O**I_0Q{~!1=iI_j3fuB2wQDtkjW)C~FngBd@F*8`;F1^#TwB>J+wr zIwCAO)|3I@>?&@2#{w>3;#1bgVrKysma9E8*IyW0?tZejZj}4m(A?=1w5K3_-Lk=`7hxxB2LX_G{uBf#zuUK>@%b;V zxBS3IYXjkYxX|VgfFuRES5+VJRn0+4*m2rWs?FU8uo7y^(^)q7&36lVmyZ4d0cBUD zm-XgkebJJ}XfBsT*UQ2dVlHAK)T@)t75nLN{XhzTOd(l2FgIMgk(8g_PpYa9?LD`qaM2q>qe8kUVB zlLVQ^i(4`j+cNZ%$QYMy6I{oS0o}EZe%e%=#2qWeybaL@aUXEz#%B+umAHMOpJ@t~ z+jw(Zo^TYi0a)n9GR;&`}KlilZKcrj%t(FoXT8>5sbyj zwN-*JemgzNa^lOazqAmMdH1YW{UkJ0*TQDxAlV^P@+gq4)@h-B^-exk*@bA!Otqym z;8#JfQSQVaMDqh@{@+oZ{j&=)iL&m?vK6rBO&_DvX(%@PD@mCl-)-GL5W(N@jrwHH zdjn`=qG1Z>oBi-TS^*C-6t6FXimu<8GMz)FoxGc!{l0lrRT zm_wc9p@GE@FQemf9KSMVKewzmYp?aQn)9u4^x-)=@X-A3PsRh4g0U+2IGn3UN?DyO z*he?`Hfu)J-7nEVGd0jNUwso+)P(JThx)+@Qzl`V{yUrU@XVrnbPuKlovkP63dhawKf|`}e%~g_+pzDp ztCxA_V#1#&Kh*$dVJ|wE$_@eguNM}@igBFknxmT1dGPxM)+xQI4zsM~9>;ozUAHGD z>!TdWjhN_X@WzzrEc($Dn_7K!j_2ty)L^oQd6N&Do4r+S`l@8dX-h;!%=wUu$7Q-`y9-)3FOT7#m1Ly65NG4nJpF&wX%(0XV9$cHYE)P+ylf-w0Gy z1v{ZX#C?kzaGc7Y_SeG3lmXWJQjCf3_evlBvFeo(QJ2SVeK@o~R;S7B%88|CbG3%Z zODHzV;@y-t)PKxGQ7;5Mud(_A{IQ<|l~BmjIOy-;b#^b5e?inHs#Ua)et2ZiQj*h4 z;tpl<#Z=2psc&+&cU`*r(Ua!c2gX{o39!!6V}?QGjhJhjYQtq1Pdn1PEej;YqW_S5Fs$SRK$?#uV^f%IQq!xGoHi{!KtSJKv+0_PVMd$>W^B`-Su z@)&!UrJ&`)L$G^2%o*VJCbH5p7P<^~=NinC%9ahL%(G>>W0$(GpWdd}*3>G%&WUrN z8qqM@>;`vZYFAFud#L_USKL7~tP`X6X?(h*;NG;7-yY|O@o7hd))K5JyZVQ%Bv{P4 ziaNHd!2?nU-0LXM)431mzetMB-S;9duB@v7RW@IU4Oua_w@h4QX!N-;|Ew2Z^G=)J z@}XK&42&bWxE_Dj!?3LC+4ua+KHGLglX(k>aWrK#`ojPS+aTfjB{Keb5g5@g4^Gcb z%o)HU#_2}~Y%js5k6r_%46Ww32}2nY%OIz|nb=@gdQ60Tmev|J);|)-y`n+GKoks% z-97et)~BS)n$@0Q~R1@BO&2XxU6 zni6FFk;l_>*O1Lmu!w^PMp}=)+E~uYu2WT(DquOUjk8rpd^R%5-;}+~^ToHqCJbM= z&GW19!M|UHZ&FMo>Mb0+~lcLo7=HL8rZVlYP1TVoUmmrSCi$oYIM@`FgV-470{c z%V8dod0FO`SpsqNLc@8@?HVj2LUMnkOLJS@{vsz+V479C&_q=Ip2J9aLwh|J*B6!S z8Nr4rqGmS1QSwbg(i1U~@MVa68i-QRAH2OQ<-EU-O}E5jeo?8}buo)muDT;(6P5F+ zU)OyZi&a~>#}@0R8q4k|4z=6J)QFHymv@sl<9)fqJCAL@Pmp3;wu@dq4||Xxfz>qR zn%u|tr7c*-5B()Q;g>lsGkaG5DshhaH{N1xPNlzPUP-pq^!PJZg~sWL=bD%MIwG2d zpxbiuru$WVKmZ=S;kg8Ser7O}jA6!Irtxgedv;Aize>LJ8SFTE*{+AxI{R=~yt-Rc z!HxZT^h~72vgS7ZJQ8}4w6M_kC}w~Zr#$X}&5zSlI{qqx?P9XYKI!RyLPgMjrye4m zcl4Y#k2X^UYVq7$^DmxSCT2I{gV`Rg`9`={Xn~{>$PKPO7k)78gQi;~?%F#|?eSC# zoSpW9mg?NLXH&O#hLe==)u-^y9Pc>pKbxEoT6xTK1x+CCa@2Z%qQ^A-^zc}kyVUt3 z*ORKYQ^lU^Bhj%3Ngr#ZR>w;SfoAXqc4yXq|D}b9j}?QG8J|YDeSJ*xUB&PsL+To0b$@%b&ot@cM*H1r-t}(@LV58kEfZ^4WP;ue zbTap!e!}8WeA7ZNTy}=KQ+FzJpeK@`tJtKEj^HW$y8^@n=PTfgTllrYNW8a)SAaxy zpoHA3C3)4f%Ne8~3iMu74KG+8aJqUV>@s98mL8qlDyo`thAD?LPiNFr+78^Ai3PPz zd;}G%Xp2+L0trWn?qd=9&gvWCPyvzh_iULgqO_weeXkdWF-KqnH3}OKI}WE>EusO$ zS(irV39Kh9EUz-^#Z9Bea=5UncRIz6{~bkqh{HewAJkB^5M%dotk27NM!d-irjLF! zqT=y9E`|XJ2yNax30Q!uiGyS1FB6rC;u%+DoAJzCCOXGi@=A+$HBuL36-TIB1jC>; z0x*RZSTg2R^qSyT=uHRVzjJp>MFpPLJ-V6<^3^24A|ASSTf#X_ zbS_g>4#mslQB-!k^rIV3G2BFRYLlpiTsdhO*_54El1eR-myT2!G}wZ}24KRN-g2bQ z{?@d6-f!ov%N7)PGPm5pIcxw=jZ{nU%vd4w2DVC;&F`G8;wq#_kqk2pLae#)4Uj*a zc!Sse`abUGr>3|TV38CG*Sj8-46Boom4c9WA&vP^&R!PZ9wOFL<(% zer$<`jWf7|Ore6ehUy5j`^e_ps1I_d3Nuc6jMPXBc*ot7ye&o{E9fPH&ml`2 zUn8!j`bqT7cOI2?(K`JNp`t00{yzUkroV^&*V8N^-@Q&l)S5eWxQ|XNRkPER{%URw zkjs6~fbc@YKHZq9HFO)<1VPQL>=mC`n%R!kaS1JJB(vc^ zk)b1R%omHPB5tS3OkM|^zYTg z{M8|m)q##m5zotkJPR4b9#^rYQ`d~mc+UtAl63i!pTY6*X;*&4>Sf0U{z@`gwK4CB zwOZTkAS-+Hhl#g0^ZLq>3vUNy-OZb>XHxmP?j~fEuCUwuxXRZ;i$Ge)X? z-S?ft(}mRGbWd;BC6M)%e|B%6kw0%!R_4p6>xoU2KAkUvAKcHcTg`Y5laCU&OpQ zI{Horv^eeuM1F`mmtM3^kOCAfZC(Zp`QJaRIozp5uICb|*O(UkN#TPf+JW>ap+pPPEwVoLAdsy(LIM^rb>~Pr1X8Q%7{& z7|A?f6+h$9r^6XOT$Kw?4vpuy5o3)rzV#;-_I$=f6Mvl!h>BH`O_F!UMk;i-djP+& zmbP~x>CVA*jXX0%~Hv9w8Azk1n1pp7KA$9^~Jtp4ACNap=`R10%)WjQLU+A&Jx zIu2fS-p=rrc66E09)TW{{N+}7c$UwyruH!I&P^x(+E<0u*Mmb=Qkv$ps0?_3&Gi+j zsY`!n@VOk`r2L$6YdZSp46Wu>bKQlvEz`NTfXsA;@%_|@A}H@vj#sV0P*(ScX7o|B$v%=3w$_o^onv)%aK9gVYbfHx=sct zxzWsPMJZ`emg_o0(^U4O6yrqx&z)jNbT-NDpUyYtGLG}8?c7HoW3aIc0ogqH?{~fJ ze|_y0z-sV~8R`D0K+2`?FSd?jczZ1Gkleg8vSL_@6!i^^scWSA-XiHV{>?hWmXbUg zfJp66#rj#MAAmfllf){N+@WY_RuEVQZjC@#=TjqBvty)le;KlF+{px)QVIYf9LP?w z@@VXvjrY^!y1Z=7=oIZL_`<@dXEz=v4wfc=XmgGwvC!cAt}bnmfLTfxv7bLe4Y603 zqwd$0qds*FT3~K0Zw5nY_ILADfSM=l?Bz=w_D8`(9(;L~Mb8E|#?w4^pdIUOThmH6 z@&q$`MMwIeH%O20<%`*&%fW<3oDV#C>(`@SJvUPh<=K9~zSZI0u`a8fxMIl~J!Dv3 zH*m>QI2_CBVeSP9B4iRK>4I}u@-_yQ5$ew8CoUs6TXXzt!O(Fpl&6z9-jHHvqw#p; zwYT@vj|>@Kb6S^b2lQQTy;TE}s~oN8p&S;i!SMdl(;M#w+?{IYjLa2x@zx$ymiQ3n z8JS~XqV(&+7W>&pb5?x%Rcx4m_kwN-xF#^2zU_(Dp+iN5CpS|2ACDgBFRK(z}bGFFS3XOCUd8 zYb%Fen;!ApX(2U~%Cv|Ydm*pO)~)t7U>t0Ku*MouWWRW|yjXmy2yMgX?ZOY()p2i} zvCv`pSv_WFcd0%1R)g`2O-ud~C?~ zS~)-I+vb`(2{QZ3M4UcM`RZ2`M_|QOBC^V5#cO^lpEqN${Ma|8C`*Iuqm0^E0?bEyHEpxJ zMH(j7r|m0B!jRI5&zs=bn!}jQHhojc-XjQOsu=N`%AB`UX!Hy(eDqEtKCoy74a*Z2 zBTRq?G$fI7u(Tuuf3$5pe7X}6XaR!v+X|W3rFO#0e769UT(Cz6%CcfxE0gb)TIH=9 z(~oA;oS9WHAQ-f12=IO+Y3^mj*G7-SUX!L=_SqO?Q4Y8-&O}7eM_B4|;F;cke%zNs zY4lZS)}SDTcQ&uae&_~v-nU}gLdBwQQGZ)-Hb2-Oj{gH$gE;&c`d#$}uBDXQiqN^Gq$crS};)Fdh3sxL8yH_*r^gL?lTv#J9 zqDp1*deqQW$uamqzhAujg4D4Vju2z%?yPU8(!PQr()m`40EaD7*OkE?H}C5o$n72RtRM0X>~LV z!OilPbF3Rz_3KG3on{@%G|BTP?A}EEd8!N#D2H*{%jt^noT+S(CS?*eQ6`+)!Po>< zG~>}tF<<)as$3KYbUI>^q?x)u#G`sGKC9banq!WH;2Fw*e_D|S0zqwMX|U%OU;UV7 zfxtdyR-e0nx3hlrlDB`rBw1(b)tG83>u>3xjLZaBVk55fWAYkBSLGa&at8DN0xzkR zMz5W`PEd$(9UO%m#=Lb%FipMZPMWpHM&QnrwMWbLgb3HgON|0tYuy)9iB4T)9v3K% zFXA?RS;IfO+A1$~ziB-Z)HemE35;VzFAJzbHEsPSp!o8xF84Lx7dd)oLqPqfuF5n0 z?X+~-r5e`OUcQnC7Ex#wVy)S(()P?WDs!jL?(31laXx1N7ce9lJ*%EOvQa}F^onz? zPKV7N4zED%6sQa^NEB`8NY?LS?S_gB-W-{Ok!oG-Sn2uEQp40B=ylAek(iyx?TbLW zstiiTz%=P=nfuC+*#l)rIR~F0y!bmactiUTu5#X^!tlzPnjk2hO9P01-Cuf5^3CiYvQg7CsH_5;gK$5K?i@uf}%Z`WUeaWwuD{eD(5 zwO(k9Wasi_G1}tEDV`#k0g6p~5Lu*(@@surCM4zd9wJ`aJUrA1$?|F}M^$LsvoSX; z?r9F_LSpt>Vj!{aTWF!x^PfX0|A^7Yg1s#kJ;vY(;j z;H>_PyZlnIaeI!8JrGH~Ep0PY>$GA8)P9A*;D(hK^qJne-()NPzg%URjq6!ft)Z<9ycQPt^@`OEZac z-&5~uLRKVZwx%9#+yxqfI2GmNI#mbA{KCymH=l8{1V7n*xkZdj0qsf=uujr8wJ7S; zP}|!!Vk`M;s4bl`%mG~1A_)}w=L{8VdGqFw5V|gZR8<8Bc6U48PxG4hrfbQlVr9KS zc;xC9Xf}&PgUZ0D*z^nKt0azHAkk@SS3wC%FbBg#4Z!$H@^L9YOakT)uGJ&a#tb9A zhd@!vQ?=?6-h@5ZGnSVtkQSCTB1u2Q;P(nU}AP9inC1#RYD7v@wkbU%IUKK z-c=Y8{7S>#Fs|Gb2P`05uJ9!gaSSb(g*+X_4;1Oq+PKL$EJEV@$*M%Z6_q694q=Av zLXV4ORxD;ja`dk@lPNiLfzaT=d51UH8r?U5iIB~1bWubR?IU@t(E`^DFjT*eZh)Nu zwW7uk6K!%=jiHz zT`Eo?)7X~;w1f>Es=?(>*S7r!fcFGGS;j~jv-;h8N14I0r5pn5I)MZK`z7{E6q{7^ zwHcf_G+MO?f7y+IDS$x$0lqqY?bhWZ zcSZ%dUn-1QKVDB_H{>4~$6XNIQ*c{X;*}rMAdOGPzn6fEEINeW1&}ba~oMUxJSW}y@I*cm+s7ybDf*;6wCaiPk4P58m$N)oi zI(H9%@kh1u78Z}%2*w$F_-xS);QKcbDD1UjhBk${5I3OI2zQba(}`tRehwB3xz|50 zGv~=M0(O^hLsui2_iSiSs6jC#qpF-m0+#q6r6TUU1yhNE#EjvK2Eps<^3JiqjR7Bd z!25pC;6@pUGpj|;*hu?>rpy0sICe7By_-+mMq!B?J`J{VAt#3D{#!oyO@XIx|0DYU z;I9s(I;cx^1|ZOt4cT#~vjf?rxaGiBr@xaY+5Z?3<7n8#^9L8u&QxbAr+?X1h@z9f zY-eHjju{8~<}TJ|&o4H9y${ek_`}>%?zo_1Zb1dvIkE36k;S<+r8$N+VM7<13X3zH zrdjLmiFyXM@;5jdXPhkr;0L-$VaOpKv*tB_`z^&9Lu5nOKTaH0ZsK0+^gyK(n}`kb z=a$168s&!^N{4p|y%6xK{!!G*R7zLeV!!gZS93nT6r ztjQGW``>DhBlSKp{^DvW_!s|ShS=w$rYZDm!2pI#HVaYX7$0|0sTN~l8NQ&>!k+J~ zf5dYz)qWP=rpRrS8zpP+yUHZ*9U-1p?eVx?svPH*}6U7Y3N@}J31 z*UgRbA7`-(ROJ@q_j&(Stkjnqn)Mu_uVZK@GY8A@}AIE-mdhD_IyDeXxn?iRy!u!6Oc-FE*bKQ_kY z?l%?DgdI9nIfs7m6fmD(Il(tR*rKXX55jlvuTyp4jA9t}lio7#I8^3nfJ_kD}syJ2qISCRP#zWIHNvOPs}CnDo@ ziioyAYN8aV=Mra{URK;yd=2}JI};Z}{OFr?Z3RwE+kVg2pthT4#`?^@kWDP266R_Z$A5<9pL>$h##O!WX|ua{Ur>CfBo|)3 zk0_6F*0aOf>iGX8*;_W+KosO^xd%l{7Bxs#R2*YJSf%dHTMUs$4tOqV3tU(%%{M<* zkvczV{hxNO{h#Uf|4)+AE#=NlO4D76A}WU)*&JF_CdXuqB*&5D6gI0AC2}gKu-jP^ z5>w2!+;!({IZd`$ISn(z*v#0z?>?V@;`?}fzwckR{c^qbzTVgC`MRFh`}KTwa!iBb z7GSOiC6lOAJ6-1*FHR4Btvhx9X?mX_ zbY_(;cLQ;n0M@|T#TG3)kJ_CIOuN#ld0C8}es6YDN7Fv70^-lFAFjPpDIZkR&#vAg zY90el{zr|5U-d*}ZytQ@FgY=P=S!|+zXwvzOi>K=NzqyFH2SOjyqX9qC+;i!B;hxf zTc(c?iu>whKnD1zqyUpK5;rP#mVYlB&&>ws`6cht4E}u{!nu#>J00TUG93|8LEo{q zwXq>Som_&0+l1FazwimtN#^iPg z#i0tW&KOE9sGl)Tlvzaow@_%<6?)aRf347`lc~`&0BzU7e^fsi2lt zZNscX<=NNC+!rf7=BBwXWkXC<@#zYj28;~(aE5k1~3uAj*JVnETtI==O*N+@N1Y|DL9l3o<5 z9LFAt46MWu+CdTa-&sA+o6c$~+NX-cb{2Xvwb05yrgj>6V*9>rl)!y*z%Ea<_Qpb^ z^2YB5#-_0P7a%|l84M3RZf8+h#>G5!e?MpjjI>c&Pc@$MknNonJX;Kwq;{#(PGh=b5la zt26YJBE=cMaPa5QaLSrbZMb{HaPP?QSs-w+bI7Oc4n8WMrnhJ8uypQUBga*J;(cio ze+{ECC$^BEBLp+_?dMc;@1P@tV~|ph9Q9x7GiFkJ5hwo51g(D#*!b``7Gu4P1e8ao z!82my+cg+Dy{)}y#M&d@#O;nGK}jR1v`o)Ee&e!3x*cw$JOiJ|y6}lDXP#5CJ1n)b zT)8n+XfPL4(Q+c5?gLtCW69X+a$+8}@YAgDXiTN}*T}U&uWFrVFaLn039VXc=_(BX z4Hz-FT?!Ygj0})WrYWdqEcWQIT2^Mb*owZl)A)5PN58ce^g{0o8aXi2$Qg1Ww5ggV zBY_CBwsHxPdpxA*yxa&5Sk{TRR=5zo4>bT?h5FczQkpGgByZ>ys4q;+!ZKXqI6gFb z%G;~wFYzW{HOA#V{MIq!MsT?NV^e?V2-?n!jr0U%Jf%b^z}UJf<=I@IdM zJW~ivp+n$sDErcriJMtL4JNB;B75P6osbBK=4=bK`h6v|h|h$oLB2)(_FYV`Z1fU2 z#|TCK<_7jV9Cu*EjX;Z^kbAH6oyuJi;I5-~Lx&9dI&)+O6E@IDsn174!o^;0uLjNC zh-aOXO8TO0EBPws`UnBTGV56`Sao!eaf&=aT~=^M}|etPF_|cfOBsfOQM*Ik>=)ya{&)J;uGR;QUb55KPj4gzk=K zj!8z~S21J!x*5!t+lq%=T6IY;qI+vxS;&2rDMF!dw=@wUeF;e{`$`TjX0uUbe#o!! z_s2{%H)JSulFV+j4kWIGEjGb0tanjdiqMC}IjCUS;=d?ecr68%eNSXuLo2JV7Om*R0 z7-7U^Hl31m%X@BJzaz*YE0h)XN-+H)#ruzp9o>a{e6-zo(Tev8)GmNouqFbj?QgFs z_VFNDJMIQR_j_os7yHQylcaRNfU=INRht~0_Oi+4y1Nkj^&^br66DD^at2Suv6`p1 z6#n;^?4MuX7JgWbLxzGoumf&&G&(cuSf@g*5(YdaSxhDHeXq9f2Kuv$Y|>FH-dk^i z5Yg(wNs%XHp`i5c$P|7~#<-}ZaIS)?oZCWc@IR(vAZ*2$eXR0#Vf5>NLsmC9YXy$* zJ@aqQxlQu)rhY}7`1-E!o`f{G#(&_FT7kW`g!!l$I|V4!(N166B3$1<>S_5vQwMU% zhHH&}Ly$yJT-ThIq^ix}NRv~-J%LQ{H(f+QDa5272b^-z*v@y)OI|RtzxCz zw`h@GHL~Zq+qN8L zq+~`-AQpLJYkcG_Q3G)>F?UtiJN8gMx{uK6}^#!Ni7>OQ?P=_(lP3LU^vqeI}dcu3$(&5Mt1Fc zfh9d>^X?F8u8P!XTHb!7bbi!hf~;`h5}kIWX=f6E%u zDJ3*EB4Xd_p%~}L--MIGAJJEmm-^kL=R2|2@*k~sQT;lZYh&bFIIsSef9%-aZ$=RiT;8QrMm2=xNlbJ9;C@Ov=w(Pr#iyQp=WSqFk z3TWn4a$f}f)ZaCS@7!qvDVQ_RXGOO^pJ7{JL^GPXDBTC6<6~`-`dpJPd2|a$)E7UQaYpTx3?>`y}mwoSiS8z<9GbS zj^$?y#xD)7fbz8t6~5@AGiStG%zg2FDZSZDX!l1jD?_6%Ov@`^NUZXubU{q zEX{z208*V|0Oe0Q45l37^@r`K^{?CZ$ggGn6S7fn~r(Mr=~Q3G3A2=1~K;^(kR!?h(1FH~2ogNqsTj8^_LdtjOSWr+;hK%j1qvTh%7aZ3Gug4hRhbYo&z%ad*8XgD_3ikIJ)3N>GTNb zO`@0pG<(?96o^=Th3UPL#hp41MBWz1JlvdT(wwwGxOoZglD(cW3R(WMEbbWiZ!Arv zMm&xX`4(DdXOvuXuY37sIpy`ak(^b_E}V?3I)z&IQ+HI^C?n&g( zhCwxO&B1Zk^i@UTiK-qy)s==kOgtrf)A)6G%D`OLcyVZg>1j*RCuI;vwX(~s?N_*m`8Y(E5tkL7%+32d<@753*)DQDs3`#ex zwM+X+36^-=Nqz3E)L3h0{FkY7IXKu0X4k1uok3$Dljh+TaZ7OkHQd37^ELHy@IcE) z9FDajD%=1uZ|VNUdTAST^~|stV6{hy54f`?Ny_({JtAkmUa$i3V|LBDADMfr1b<*_ z8tn0#)I8A{T6!bjQ@qWWZy0HE)Cw-DrblUGZ2_QwT+e|lX+LPOEJ*U~UzAq)h;Xsj z6^cvoaR7oAb*MI>D)LZ5v@jA(Pr-XK3IeFd?QtcJ>U+H ziDs3TaCp6<7?mH!Woeg)3O`7hyW?wiygPQT2mdM4MgL~sf2+on_0D8ks`3_cc<6&k z+_7sy(k(TAOR46Y1D#Y{oyp^Zc%3{Q3xE093>5FXCb~9!$o<|a2a{gn4N-kr8|3Hc z?zd+=?#V|U-eCcNUOQ(Z>(p#^U76==HizA%tn-IDxRJSueC2)w}acA zMcS?pwge9~Ec*l9Y~J+Rv;;k%TIm7#bRY1tSnEntZ`I1%aC9aA{*xxc(85gLziz(v zHgzzq`G6vfm9_%n2#})?irL7U|FZu=_3LqND!lZ!tSlqQIc9rWM2_ zr&XD;sVfWt4F!9T>$(;UL#c@_EbmcIS@5s!V+B9?F?2k@a91P38ilbPOo?HJ0e6H9 zs}KDLK*cH3V=xr?naQD?ZDk8MPZyc@XAYdH3r{&RZK?xSe#0Q@C*1fVIUytJbq(PS zA})ECZW?RPKN)b7_Up!ylkssZXsF8(QWc5RxZ7Hx?g|;c9&Pb26TMtt(o}C-$W5#b zsY{D`&yHDI-s)EYc1ka=D9avf`W+d{O+9~GT%U9}!ZHo`ANxbVJgbeL**b130P`PO z039ph3rNm{)@hTrreu>WXj!}Z+m=#THRIAg6BEU~BIMgOy$H z_h(U~93FJz4fRFBlR;i&z>zC5r>Ph})>WINI~ElT(7Eq_^8|Oltdtvua7E$l*JGU6 z!zN})iU6lOD^eX?ln&EGoe*maTgjaS7ohOKeB-8s^?Ri*4wz+>+=i2Lnfo96G$V{q z`pqvh_vMV!HZK``Kd!tz{v~Fdm%k9R>C^r&PWkL$AvyGpM#CIWrlPi!HC^5qsElaT z^`s}BJgd6WHjC+(aW&?<4SuS2P89yZnh9(RYq}kcDUiKNE!F{MYMTz@sUXB|gds~~ z`B+F0Vi!^zG$L&Wk-g z=i>P*`CFwIlQ|J`LM8pXrbt; zW!iw0&kIp73rf-TCnzZPk+lO8h)weF;k*eXKa^lSFfF<@!^;c1P0qO=GdE#9!w+E~ zLs`Y_@PKDg;3w9pQ4`&G9TVdCcqz8qCkAMexc4C$-wH_HflKov&=C6T`hWB4|MN!P yVyr0A7FY`R^#Zn>+&@VpfbRwAzg|4vvzIrVQMxa0$W{V=fiA)9F4S7yc=$h|1cF2W literal 0 HcmV?d00001 diff --git a/platform/micro-services/website/source/react/logo.svg b/platform/micro-services/website/source/react/logo.svg new file mode 100644 index 00000000..9dfc1c05 --- /dev/null +++ b/platform/micro-services/website/source/react/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/platform/micro-services/website/source/react/reportWebVitals.js b/platform/micro-services/website/source/react/reportWebVitals.js new file mode 100644 index 00000000..5253d3ad --- /dev/null +++ b/platform/micro-services/website/source/react/reportWebVitals.js @@ -0,0 +1,13 @@ +const reportWebVitals = onPerfEntry => { + if (onPerfEntry && onPerfEntry instanceof Function) { + import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { + getCLS(onPerfEntry); + getFID(onPerfEntry); + getFCP(onPerfEntry); + getLCP(onPerfEntry); + getTTFB(onPerfEntry); + }); + } +}; + +export default reportWebVitals; diff --git a/platform/micro-services/website/source/react/setupTests.js b/platform/micro-services/website/source/react/setupTests.js new file mode 100644 index 00000000..8f2609b7 --- /dev/null +++ b/platform/micro-services/website/source/react/setupTests.js @@ -0,0 +1,5 @@ +// jest-dom adds custom jest matchers for asserting on DOM nodes. +// allows you to do things like: +// expect(element).toHaveTextContent(/react/i) +// learn more: https://github.com/testing-library/jest-dom +import '@testing-library/jest-dom'; diff --git a/platform/micro-services/website/start-dev b/platform/micro-services/website/start-dev new file mode 100755 index 00000000..cea26de8 --- /dev/null +++ b/platform/micro-services/website/start-dev @@ -0,0 +1,66 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH="$LOCAL_DIRECTORY/source/flask" +export PYTHONPATH + +# Setup google credentials +export GOOGLE_APPLICATION_CREDENTIALS=$LOCAL_DIRECTORY/source/config/gcloud_key.json + +# export flask +export FLASK_APP=$LOCAL_DIRECTORY/source/flask/app.py + +# Setup the react dev environment +REACT_DEV_ENVIRONMENT=$LOCAL_DIRECTORY/react-development + +# Setup the dev environment if it doesn't exist +if [ ! -d $REACT_DEV_ENVIRONMENT ]; then +npx create-react-app $REACT_DEV_ENVIRONMENT +fi + +# Move the code over +rsync -av --delete $LOCAL_DIRECTORY/source/react/ $REACT_DEV_ENVIRONMENT/src/ +rsync -av --delete $LOCAL_DIRECTORY/source/config/ $REACT_DEV_ENVIRONMENT/src/config/ + +# kill background tasks on script exit +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +# Start the dev environment +cd $REACT_DEV_ENVIRONMENT +python -m flask run & +COLOR=1 npm start | cat & +wait + + diff --git a/platform/peoples-speech b/platform/peoples-speech new file mode 100755 index 00000000..437320a7 --- /dev/null +++ b/platform/peoples-speech @@ -0,0 +1,51 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r requirements.txt > /dev/null + +# Set bash environment +PYTHONPATH="$LOCAL_DIRECTORY/micro-services/cloud-hal/source" +PYTHONPATH+=":$LOCAL_DIRECTORY/micro-services/data-book/source" +PYTHONPATH+=":$LOCAL_DIRECTORY/micro-services/peoples-speech/source" +PYTHONPATH+=":$LOCAL_DIRECTORY/micro-services/task-manager/source" +PYTHONPATH+=":$LOCAL_DIRECTORY/micro-services/ingestion/source" +PYTHONPATH+=":$LOCAL_DIRECTORY/micro-services/quality/source" +PYTHONPATH+=":$LOCAL_DIRECTORY/micro-services/data-export/source" +PYTHONPATH+=":$LOCAL_DIRECTORY/micro-services/forced-alignment/source" +PYTHONPATH+=":$LOCAL_DIRECTORY/micro-services/search/source" +export PYTHONPATH + +# Launch it +python $LOCAL_DIRECTORY/micro-services/command-line-interface/source/cli/peoples-speech.py "$@" + + + diff --git a/platform/provisioning/gcloud-ubuntu-20.4/provision.sh b/platform/provisioning/gcloud-ubuntu-20.4/provision.sh new file mode 100644 index 00000000..e69de29b diff --git a/platform/requirements.txt b/platform/requirements.txt new file mode 100644 index 00000000..fd3a245b --- /dev/null +++ b/platform/requirements.txt @@ -0,0 +1,6 @@ +iso-639 +smart-open +gtts +python-configuration[yaml] +whoosh +audiofile From d65e604872296a96e4af13ac5891a5285bdd9eff Mon Sep 17 00:00:00 2001 From: greg-landing Date: Tue, 27 Apr 2021 16:35:32 -0700 Subject: [PATCH 2/3] parent bc676643bc73bdd64407b011ee760f6a2b7639d3 author greg-landing 1619566532 -0700 committer greg-landing 1621617348 -0700 parent bc676643bc73bdd64407b011ee760f6a2b7639d3 author greg-landing 1619566532 -0700 committer greg-landing 1621617229 -0700 ENH: Added parallel data exporter to run on kubernetes cluster. BUG: Missing config param BUG: Missing key BUG: Encoding BUG: Encoding BUG: Encoding BUG: Encoding BUG: Encoding BUG: Encoding BUG: Missing copy back to gcs. BUG: Encoding BUG: Encoding TEST: Added unit test for data export. ENH: Cloudbuild wwork. ENH: REST data export testing. ENH: Added parallel data exporter to run on kubernetes cluster. BUG: Missing config param BUG: Encoding BUG: Encoding TEST: Added unit test for data export. ENH: Cloudbuild wwork. --- .../source/cli/peoples-speech.py | 20 ++- .../flask/cloud-build/cloudbuild.sh | 13 ++ .../flask/cloud-build/cloudbuild.yaml | 14 ++ .../build-scripts/flask/docker/Dockerfile | 9 ++ .../flask/docker/build-container.sh | 14 ++ .../flask/kubernetes/deploy-staging.sh | 2 + .../flask/kubernetes/export-service.yaml | 50 +++++++ .../flask/kubernetes/run-export-service.yaml | 21 +++ .../data-export/requirements.txt | 7 + platform/micro-services/data-export/run-tests | 41 ++++++ .../source/{ => commands}/add-clean-text.py | 0 .../{ => commands}/add-duration-metadata.py | 0 .../add-google-speech-transcript.py | 0 .../add-librispeech-metadata.py | 0 .../{ => commands}/convert-audio-format.py | 0 .../{ => commands}/convert-cc-search.py | 0 .../{ => commands}/convert-common-voice.py | 0 .../convert-dipco-dataset-format.py | 0 .../{ => commands}/convert-librispeech.py | 0 .../convert-librivox-google-cloud.py | 0 .../source/{ => commands}/convert-librivox.py | 0 .../convert-timit-dataset-format.py | 0 .../source/{ => commands}/convert-voicery.py | 0 .../{ => commands}/convert-warc-to-csv.py | 0 .../source/{ => commands}/count-duration.py | 0 .../make-librivox-train-test-split.py | 0 .../make-peoples-speech-train-test-splits.py | 0 .../{ => commands}/make-tar-google-cloud.py | 0 .../data-export/source/configs/default.yaml | 6 + .../data-export/source/configs/local.yaml | 10 ++ .../data-export/source/configs/server.yaml | 10 ++ .../source/data_export/__init__.py | 4 +- .../source/data_export/api/export_dataset.py | 13 ++ .../data_export/api/export_dataset_by_id.py | 10 ++ .../data_export/api/setup_cli_parser.py | 80 ++++++++++ .../api/test/test_export_dataset.py | 36 +++++ .../source/data_export/database/__init__.py | 4 + .../database/get_dataset_from_id.py | 10 ++ .../source/data_export/export/__init__.py | 4 + .../data_export/export/cloud_exporter.py | 31 ++++ .../data_export/export/exporter_factory.py | 20 +++ .../export/google_cloud_parallel_exporter.py | 30 ++++ .../data_export/export/local_exporter.py | 13 ++ .../data_export/google/export_worker.py | 41 ++++++ .../google/google_cloud_work_queue.py | 138 ++++++++++++++++++ .../data_export/google/redis_service.py | 51 +++++++ .../source/data_export/utility/__init__.py | 4 + .../source/data_export/utility/get_config.py | 47 ++++++ .../data_export/utility/load_dataset_csv.py | 31 ++++ .../utility/write_samples_to_tar_gz.py | 51 +++++++ .../data-export/source/flask/app.py | 18 +++ platform/micro-services/data-export/start-dev | 51 +++++++ .../data-export/start-production | 51 +++++++ .../peoples_speech/data_export/__init__.py | 1 + .../build-scripts/cloud-build/cloudbuild.sh | 1 + .../build-scripts/cloud-build/cloudbuild.yaml | 3 + .../website/build-scripts/docker/Dockerfile | 8 + .../build-scripts/docker/build-container.sh | 14 ++ .../kubernetes/deploy-staging.sh | 2 + .../micro-services/website/start-production | 66 +++++++++ platform/peoples-speech | 4 +- platform/requirements.txt | 1 + 62 files changed, 1050 insertions(+), 5 deletions(-) create mode 100755 platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.sh create mode 100644 platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.yaml create mode 100644 platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile create mode 100755 platform/micro-services/data-export/build-scripts/flask/docker/build-container.sh create mode 100644 platform/micro-services/data-export/build-scripts/flask/kubernetes/deploy-staging.sh create mode 100644 platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml create mode 100644 platform/micro-services/data-export/build-scripts/flask/kubernetes/run-export-service.yaml create mode 100644 platform/micro-services/data-export/requirements.txt create mode 100755 platform/micro-services/data-export/run-tests rename platform/micro-services/data-export/source/{ => commands}/add-clean-text.py (100%) rename platform/micro-services/data-export/source/{ => commands}/add-duration-metadata.py (100%) rename platform/micro-services/data-export/source/{ => commands}/add-google-speech-transcript.py (100%) rename platform/micro-services/data-export/source/{ => commands}/add-librispeech-metadata.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-audio-format.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-cc-search.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-common-voice.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-dipco-dataset-format.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-librispeech.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-librivox-google-cloud.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-librivox.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-timit-dataset-format.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-voicery.py (100%) rename platform/micro-services/data-export/source/{ => commands}/convert-warc-to-csv.py (100%) rename platform/micro-services/data-export/source/{ => commands}/count-duration.py (100%) rename platform/micro-services/data-export/source/{ => commands}/make-librivox-train-test-split.py (100%) rename platform/micro-services/data-export/source/{ => commands}/make-peoples-speech-train-test-splits.py (100%) rename platform/micro-services/data-export/source/{ => commands}/make-tar-google-cloud.py (100%) create mode 100644 platform/micro-services/data-export/source/configs/default.yaml create mode 100644 platform/micro-services/data-export/source/configs/local.yaml create mode 100644 platform/micro-services/data-export/source/configs/server.yaml create mode 100644 platform/micro-services/data-export/source/data_export/api/export_dataset.py create mode 100644 platform/micro-services/data-export/source/data_export/api/export_dataset_by_id.py create mode 100644 platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py create mode 100644 platform/micro-services/data-export/source/data_export/api/test/test_export_dataset.py create mode 100644 platform/micro-services/data-export/source/data_export/database/__init__.py create mode 100644 platform/micro-services/data-export/source/data_export/database/get_dataset_from_id.py create mode 100644 platform/micro-services/data-export/source/data_export/export/__init__.py create mode 100644 platform/micro-services/data-export/source/data_export/export/cloud_exporter.py create mode 100644 platform/micro-services/data-export/source/data_export/export/exporter_factory.py create mode 100644 platform/micro-services/data-export/source/data_export/export/google_cloud_parallel_exporter.py create mode 100644 platform/micro-services/data-export/source/data_export/export/local_exporter.py create mode 100644 platform/micro-services/data-export/source/data_export/google/export_worker.py create mode 100644 platform/micro-services/data-export/source/data_export/google/google_cloud_work_queue.py create mode 100644 platform/micro-services/data-export/source/data_export/google/redis_service.py create mode 100644 platform/micro-services/data-export/source/data_export/utility/__init__.py create mode 100644 platform/micro-services/data-export/source/data_export/utility/get_config.py create mode 100644 platform/micro-services/data-export/source/data_export/utility/load_dataset_csv.py create mode 100644 platform/micro-services/data-export/source/data_export/utility/write_samples_to_tar_gz.py create mode 100644 platform/micro-services/data-export/source/flask/app.py create mode 100755 platform/micro-services/data-export/start-dev create mode 100755 platform/micro-services/data-export/start-production create mode 100755 platform/micro-services/website/build-scripts/cloud-build/cloudbuild.sh create mode 100644 platform/micro-services/website/build-scripts/cloud-build/cloudbuild.yaml create mode 100644 platform/micro-services/website/build-scripts/docker/Dockerfile create mode 100755 platform/micro-services/website/build-scripts/docker/build-container.sh create mode 100644 platform/micro-services/website/build-scripts/kubernetes/deploy-staging.sh create mode 100755 platform/micro-services/website/start-production diff --git a/platform/micro-services/command-line-interface/source/cli/peoples-speech.py b/platform/micro-services/command-line-interface/source/cli/peoples-speech.py index ab16d058..d0140e3d 100644 --- a/platform/micro-services/command-line-interface/source/cli/peoples-speech.py +++ b/platform/micro-services/command-line-interface/source/cli/peoples-speech.py @@ -16,13 +16,29 @@ def main(): parser = ArgumentParser("The MLCommons data engineering framework.") + subparsers = parser.add_subparsers() + + setup_cli_parser(subparsers) + peoples_speech.data_export.setup_cli_parser(subparsers) + + args = parser.parse_args() + + args.func(args) + +def setup_cli_parser(subparsers): + + parser = subparsers.add_parser('dataset') + parser.add_argument("-i", "--data-book-path", default=sample_databook_path(), help="Path to data book to generate a dataset for.") parser.add_argument("-o", "--output-dataset-path", default="", help="The path to save the new dataset.") - parser.add_argument("-c", "--config-file-path", default=".csv", help="The path to save the new dataset.") + parser.add_argument("-c", "--config-file-path", default=".csv", help="The path to the config file.") parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") - arguments = vars(parser.parse_args()) + parser.set_defaults(func=dispatch) + +def dispatch(args): + arguments = vars(args) config = setup_config(arguments) diff --git a/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.sh b/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.sh new file mode 100755 index 00000000..90fb70f4 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.sh @@ -0,0 +1,13 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +gcloud builds submit --config cloudbuild.yaml $LOCAL_DIRECTORY/../../../../../.. diff --git a/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.yaml b/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.yaml new file mode 100644 index 00000000..662f159e --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.yaml @@ -0,0 +1,14 @@ +steps: +# build the data export container +- name: 'gcr.io/cloud-builders/docker' + args: [ 'build', '-t', 'gcr.io/the-peoples-speech/data-export:0.12', '-f', 'platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile', 'platform/micro-services/data-export' ] +# push container image +- name: "gcr.io/cloud-builders/docker" + args: ["push", "gcr.io/the-peoples-speech/data-export:0.12"] +# deploy container image to GKE staging +- name: "gcr.io/cloud-builders/gke-deploy" + args: + - run + - --filename=platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml + - --location=europe-west4-c + - --cluster=peoples-speech-platform diff --git a/platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile b/platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile new file mode 100644 index 00000000..b8be3999 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile @@ -0,0 +1,9 @@ +FROM python + +COPY . /app + +ENV BOTO_CONFIG=/app/source/configs/google.boto + +EXPOSE 5000 + +CMD /app/start-production $PORT diff --git a/platform/micro-services/data-export/build-scripts/flask/docker/build-container.sh b/platform/micro-services/data-export/build-scripts/flask/docker/build-container.sh new file mode 100755 index 00000000..b75fa3e6 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/docker/build-container.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +docker build -t data-export:0.1 -f $LOCAL_DIRECTORY/Dockerfile $LOCAL_DIRECTORY/../../../../../.. + diff --git a/platform/micro-services/data-export/build-scripts/flask/kubernetes/deploy-staging.sh b/platform/micro-services/data-export/build-scripts/flask/kubernetes/deploy-staging.sh new file mode 100644 index 00000000..78874bb2 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/kubernetes/deploy-staging.sh @@ -0,0 +1,2 @@ +gcloud container clusters get-credentials peoples-speech-platform +kubectl create deployment data-export --image=gcr.io/peoples-speech/data-export:latest diff --git a/platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml b/platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml new file mode 100644 index 00000000..05b5fb7c --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: data-export + name: data-export +spec: + replicas: 1 + selector: + matchLabels: + app: data-export + template: + metadata: + labels: + app: data-export + spec: + containers: + - image: gcr.io/the-peoples-speech/data-export:0.12 + name: data-export + ports: + - containerPort: 5000 + name: tcp-c-5000 + env: + - name: "PORT" + value: "5000" + volumeMounts: + - mountPath: "/app/credentials" + name: gcloud-service-account-key + readOnly: true + volumes: + - name: gcloud-service-account-key + secret: + secretName: gcloud-service-account-key +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: data-export + name: data-export +spec: + selector: + app: data-export + ports: + - port: 5000 + targetPort: 5000 + protocol: TCP + name: tcp-5000 + type: LoadBalancer + diff --git a/platform/micro-services/data-export/build-scripts/flask/kubernetes/run-export-service.yaml b/platform/micro-services/data-export/build-scripts/flask/kubernetes/run-export-service.yaml new file mode 100644 index 00000000..3c0d7a46 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/kubernetes/run-export-service.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: data-export + name: data-export +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: data-export + template: + metadata: + labels: + app.kubernetes.io/name: data-export + spec: + containers: + - image: gcr.io/peoples-speech/data-export + name: data-export + ports: + - containerPort: 8080 diff --git a/platform/micro-services/data-export/requirements.txt b/platform/micro-services/data-export/requirements.txt new file mode 100644 index 00000000..2e86cc7c --- /dev/null +++ b/platform/micro-services/data-export/requirements.txt @@ -0,0 +1,7 @@ +flask +flask_cors +python-configuration[yaml] +smart_open[gcs] +redis +gsutil +google-compute-engine diff --git a/platform/micro-services/data-export/run-tests b/platform/micro-services/data-export/run-tests new file mode 100755 index 00000000..82f0e4a1 --- /dev/null +++ b/platform/micro-services/data-export/run-tests @@ -0,0 +1,41 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH+="$LOCAL_DIRECTORY/source" +export PYTHONPATH + +python source/data_export/api/test/test_export_dataset.py + + diff --git a/platform/micro-services/data-export/source/add-clean-text.py b/platform/micro-services/data-export/source/commands/add-clean-text.py similarity index 100% rename from platform/micro-services/data-export/source/add-clean-text.py rename to platform/micro-services/data-export/source/commands/add-clean-text.py diff --git a/platform/micro-services/data-export/source/add-duration-metadata.py b/platform/micro-services/data-export/source/commands/add-duration-metadata.py similarity index 100% rename from platform/micro-services/data-export/source/add-duration-metadata.py rename to platform/micro-services/data-export/source/commands/add-duration-metadata.py diff --git a/platform/micro-services/data-export/source/add-google-speech-transcript.py b/platform/micro-services/data-export/source/commands/add-google-speech-transcript.py similarity index 100% rename from platform/micro-services/data-export/source/add-google-speech-transcript.py rename to platform/micro-services/data-export/source/commands/add-google-speech-transcript.py diff --git a/platform/micro-services/data-export/source/add-librispeech-metadata.py b/platform/micro-services/data-export/source/commands/add-librispeech-metadata.py similarity index 100% rename from platform/micro-services/data-export/source/add-librispeech-metadata.py rename to platform/micro-services/data-export/source/commands/add-librispeech-metadata.py diff --git a/platform/micro-services/data-export/source/convert-audio-format.py b/platform/micro-services/data-export/source/commands/convert-audio-format.py similarity index 100% rename from platform/micro-services/data-export/source/convert-audio-format.py rename to platform/micro-services/data-export/source/commands/convert-audio-format.py diff --git a/platform/micro-services/data-export/source/convert-cc-search.py b/platform/micro-services/data-export/source/commands/convert-cc-search.py similarity index 100% rename from platform/micro-services/data-export/source/convert-cc-search.py rename to platform/micro-services/data-export/source/commands/convert-cc-search.py diff --git a/platform/micro-services/data-export/source/convert-common-voice.py b/platform/micro-services/data-export/source/commands/convert-common-voice.py similarity index 100% rename from platform/micro-services/data-export/source/convert-common-voice.py rename to platform/micro-services/data-export/source/commands/convert-common-voice.py diff --git a/platform/micro-services/data-export/source/convert-dipco-dataset-format.py b/platform/micro-services/data-export/source/commands/convert-dipco-dataset-format.py similarity index 100% rename from platform/micro-services/data-export/source/convert-dipco-dataset-format.py rename to platform/micro-services/data-export/source/commands/convert-dipco-dataset-format.py diff --git a/platform/micro-services/data-export/source/convert-librispeech.py b/platform/micro-services/data-export/source/commands/convert-librispeech.py similarity index 100% rename from platform/micro-services/data-export/source/convert-librispeech.py rename to platform/micro-services/data-export/source/commands/convert-librispeech.py diff --git a/platform/micro-services/data-export/source/convert-librivox-google-cloud.py b/platform/micro-services/data-export/source/commands/convert-librivox-google-cloud.py similarity index 100% rename from platform/micro-services/data-export/source/convert-librivox-google-cloud.py rename to platform/micro-services/data-export/source/commands/convert-librivox-google-cloud.py diff --git a/platform/micro-services/data-export/source/convert-librivox.py b/platform/micro-services/data-export/source/commands/convert-librivox.py similarity index 100% rename from platform/micro-services/data-export/source/convert-librivox.py rename to platform/micro-services/data-export/source/commands/convert-librivox.py diff --git a/platform/micro-services/data-export/source/convert-timit-dataset-format.py b/platform/micro-services/data-export/source/commands/convert-timit-dataset-format.py similarity index 100% rename from platform/micro-services/data-export/source/convert-timit-dataset-format.py rename to platform/micro-services/data-export/source/commands/convert-timit-dataset-format.py diff --git a/platform/micro-services/data-export/source/convert-voicery.py b/platform/micro-services/data-export/source/commands/convert-voicery.py similarity index 100% rename from platform/micro-services/data-export/source/convert-voicery.py rename to platform/micro-services/data-export/source/commands/convert-voicery.py diff --git a/platform/micro-services/data-export/source/convert-warc-to-csv.py b/platform/micro-services/data-export/source/commands/convert-warc-to-csv.py similarity index 100% rename from platform/micro-services/data-export/source/convert-warc-to-csv.py rename to platform/micro-services/data-export/source/commands/convert-warc-to-csv.py diff --git a/platform/micro-services/data-export/source/count-duration.py b/platform/micro-services/data-export/source/commands/count-duration.py similarity index 100% rename from platform/micro-services/data-export/source/count-duration.py rename to platform/micro-services/data-export/source/commands/count-duration.py diff --git a/platform/micro-services/data-export/source/make-librivox-train-test-split.py b/platform/micro-services/data-export/source/commands/make-librivox-train-test-split.py similarity index 100% rename from platform/micro-services/data-export/source/make-librivox-train-test-split.py rename to platform/micro-services/data-export/source/commands/make-librivox-train-test-split.py diff --git a/platform/micro-services/data-export/source/make-peoples-speech-train-test-splits.py b/platform/micro-services/data-export/source/commands/make-peoples-speech-train-test-splits.py similarity index 100% rename from platform/micro-services/data-export/source/make-peoples-speech-train-test-splits.py rename to platform/micro-services/data-export/source/commands/make-peoples-speech-train-test-splits.py diff --git a/platform/micro-services/data-export/source/make-tar-google-cloud.py b/platform/micro-services/data-export/source/commands/make-tar-google-cloud.py similarity index 100% rename from platform/micro-services/data-export/source/make-tar-google-cloud.py rename to platform/micro-services/data-export/source/commands/make-tar-google-cloud.py diff --git a/platform/micro-services/data-export/source/configs/default.yaml b/platform/micro-services/data-export/source/configs/default.yaml new file mode 100644 index 00000000..2d3b18ac --- /dev/null +++ b/platform/micro-services/data-export/source/configs/default.yaml @@ -0,0 +1,6 @@ +exporter: + type: CloudExporter + endpoint: http://34.91.68.228 + +verbose: True + diff --git a/platform/micro-services/data-export/source/configs/local.yaml b/platform/micro-services/data-export/source/configs/local.yaml new file mode 100644 index 00000000..e5278a66 --- /dev/null +++ b/platform/micro-services/data-export/source/configs/local.yaml @@ -0,0 +1,10 @@ + +datasets: + "0": gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.csv + +exporter: + type: LocalExporter + +output_dataset_path: gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.tar.gz + +verbose: True diff --git a/platform/micro-services/data-export/source/configs/server.yaml b/platform/micro-services/data-export/source/configs/server.yaml new file mode 100644 index 00000000..956e69b2 --- /dev/null +++ b/platform/micro-services/data-export/source/configs/server.yaml @@ -0,0 +1,10 @@ +exporter: + type: GoogleCloudParallelExporter + task_count: 1024 + +google: + work_queue: + name: "export_work_queue" + +verbose: True + diff --git a/platform/micro-services/data-export/source/data_export/__init__.py b/platform/micro-services/data-export/source/data_export/__init__.py index b0596091..9ca01754 100644 --- a/platform/micro-services/data-export/source/data_export/__init__.py +++ b/platform/micro-services/data-export/source/data_export/__init__.py @@ -1,5 +1,7 @@ from data_export.api.save_dataset import save_dataset - +from data_export.api.export_dataset_by_id import export_dataset_by_id +from data_export.api.export_dataset import export_dataset +from data_export.api.setup_cli_parser import setup_cli_parser diff --git a/platform/micro-services/data-export/source/data_export/api/export_dataset.py b/platform/micro-services/data-export/source/data_export/api/export_dataset.py new file mode 100644 index 00000000..387daa3c --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/export_dataset.py @@ -0,0 +1,13 @@ + +from data_export.export import ExporterFactory +from data_export.utility.get_config import get_config + +<<<<<<< HEAD +def export_dataset(output_dataset, dataset, config = get_config()): + exporter = ExporterFactory(config).create() + return exporter.export(output_dataset, dataset) +======= +def export_dataset(dataset, config = get_config()): + exporter = ExporterFactory(config).create() + exporter.export(dataset) +>>>>>>> d17bf137... TEST: Added unit test for data export. diff --git a/platform/micro-services/data-export/source/data_export/api/export_dataset_by_id.py b/platform/micro-services/data-export/source/data_export/api/export_dataset_by_id.py new file mode 100644 index 00000000..8605488f --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/export_dataset_by_id.py @@ -0,0 +1,10 @@ + +from data_export.export import ExporterFactory +from data_export.database import get_dataset_from_id +from data_export.utility.get_config import get_config + +def export_dataset_by_id(dataset_id, config = get_config()): + exporter = ExporterFactory(config).create() + dataset = get_dataset_from_id(config, dataset_id) + exporter.export(dataset) + diff --git a/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py b/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py new file mode 100644 index 00000000..dcb46ef9 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py @@ -0,0 +1,80 @@ +from data_export.api.export_dataset import export_dataset + +import config +import logging +import os + +logger = logging.getLogger(__name__) + + +def setup_cli_parser(subparsers): + + parser = subparsers.add_parser('export') + +<<<<<<< HEAD + parser.add_argument("-i", "--input-dataset", default="gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.csv", help="The dataset to export.") + parser.add_argument("-o", "--output-dataset-path", default="gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.tar.gz", help="The path to save the new dataset.") + parser.add_argument("-c", "--config-file-path", default="", help="The path to the config file.") +======= + parser.add_argument("-i", "--input-dataset", default="gs://the-peoples-speech-west-europe/peoples-speech-v0.5/train.csv", help="The dataset to export.") + parser.add_argument("-o", "--output-dataset-path", default="", help="The path to save the new dataset.") + parser.add_argument("-c", "--config-file-path", default=".csv", help="The path to the config file.") +>>>>>>> d17bf137... TEST: Added unit test for data export. + parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") + parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") + + parser.set_defaults(func=dispatch) + +def dispatch(args): + arguments = vars(args) + + config = setup_config(arguments) + + setup_logging(config) + + logger.debug("Full config: " + str(config)) + +<<<<<<< HEAD + export_dataset(config["output_dataset_path"], config["input_dataset"], config) +======= + export_dataset(config["input_dataset"], config) +>>>>>>> d17bf137... TEST: Added unit test for data export. + +def setup_config(dictionary): + return config.ConfigurationSet( + config.config_from_env(prefix="MLCOMMONS"), + config.config_from_yaml(config_path(dictionary), read_from_file=True), + config.config_from_dict(dictionary), + ) + +def config_path(dictionary): + if os.path.exists(dictionary["config_file_path"]): + return dictionary["config_file_path"] + + home = os.path.expanduser("~") + home_config_path = os.path.join(home, ".mlcommons", "config.yaml") + if os.path.exists(home_config_path): + return home_config_path + + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "configs", "default.yaml") + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + diff --git a/platform/micro-services/data-export/source/data_export/api/test/test_export_dataset.py b/platform/micro-services/data-export/source/data_export/api/test/test_export_dataset.py new file mode 100644 index 00000000..932a0c21 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/test/test_export_dataset.py @@ -0,0 +1,36 @@ +import unittest +import tarfile +import io +from data_export.api.export_dataset import export_dataset +from data_export.utility.load_dataset_csv import load_dataset_csv +from data_export.utility.load_dataset_csv import load_dataset_csv_from_file + +class TestDataExport(unittest.TestCase): + + def test_tiny_local(self): + tiny_dataset = "gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.csv" + tiny_config = { + "exporter" : { "type" : "LocalExporter" } } + + export_dataset("/tmp/export-data/unittest.tar.gz", tiny_dataset, tiny_config) + + self.extract_dataset(tiny_config) + self.verify_dataset(tiny_dataset, tiny_config) + + def extract_dataset(self, config): + with tarfile.open(config["output_dataset_path"], "r:gz") as tar: + members = tar.getmembers() + + def verify_dataset(self, dataset, config): + + samples = load_dataset_csv(dataset, task_id=0, task_count=1) + with tarfile.open(config["output_dataset_path"], "r:gz") as tar: + samples_file = io.TextIOWrapper(tar.extractfile("export-data/dataset.csv")) + loaded_samples = load_dataset_csv_from_file(samples_file, task_id=0, task_count=1) + + for original_sample, new_sample in zip(samples, loaded_samples): + self.assertEqual(original_sample, new_sample) + + +if __name__ == '__main__': + unittest.main() diff --git a/platform/micro-services/data-export/source/data_export/database/__init__.py b/platform/micro-services/data-export/source/data_export/database/__init__.py new file mode 100644 index 00000000..cfeed4f8 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/database/__init__.py @@ -0,0 +1,4 @@ + +from data_export.database.get_dataset_from_id import get_dataset_from_id + + diff --git a/platform/micro-services/data-export/source/data_export/database/get_dataset_from_id.py b/platform/micro-services/data-export/source/data_export/database/get_dataset_from_id.py new file mode 100644 index 00000000..2509e106 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/database/get_dataset_from_id.py @@ -0,0 +1,10 @@ + +def get_dataset_from_id(config, dataset_id): + print(config, config["datasets"].as_dict(), dataset_id) + + if dataset_id in config["datasets"].as_dict(): + return config["datasets"][dataset_id] + + assert False, "Not implemented" + + diff --git a/platform/micro-services/data-export/source/data_export/export/__init__.py b/platform/micro-services/data-export/source/data_export/export/__init__.py new file mode 100644 index 00000000..327771a8 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/__init__.py @@ -0,0 +1,4 @@ + +from data_export.export.exporter_factory import ExporterFactory + + diff --git a/platform/micro-services/data-export/source/data_export/export/cloud_exporter.py b/platform/micro-services/data-export/source/data_export/export/cloud_exporter.py new file mode 100644 index 00000000..9fda1847 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/cloud_exporter.py @@ -0,0 +1,31 @@ + +import requests + +import logging + +logger = logging.getLogger() + +class CloudExporter: + def __init__(self, config): + self.config = config + + def export(self, path, dataset): + url = self.config["exporter"]["endpoint"] + ":" + self.get_port() + "/peoples_speech/export_dataset" + data = { + "dataset" : dataset, + "output_dataset" : path + } + + logger.debug("Submitting POST request to url " + str(url)) + logger.debug(" with data " + str(data)) + response = requests.post(url, json=data) + + if response.status_code == 200: + print("Success") + print(response.json()) + else: + print("Failed to submit with error: " + str(response.status_code)) + + def get_port(self): + return "5000" + diff --git a/platform/micro-services/data-export/source/data_export/export/exporter_factory.py b/platform/micro-services/data-export/source/data_export/export/exporter_factory.py new file mode 100644 index 00000000..fcde7833 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/exporter_factory.py @@ -0,0 +1,20 @@ + +from data_export.export.google_cloud_parallel_exporter import GoogleCloudParallelExporter +from data_export.export.local_exporter import LocalExporter +from data_export.export.cloud_exporter import CloudExporter + +class ExporterFactory: + def __init__(self, config): + self.config = config + + def create(self): + if self.config["exporter"]["type"] == "GoogleCloudParallelExporter": + return GoogleCloudParallelExporter(self.config) + if self.config["exporter"]["type"] == "LocalExporter": + return LocalExporter(self.config) + if self.config["exporter"]["type"] == "CloudExporter": + return CloudExporter(self.config) + + assert False, "Unknown exporter type '" + self.config["exporter"]["type"] + "'" + + diff --git a/platform/micro-services/data-export/source/data_export/export/google_cloud_parallel_exporter.py b/platform/micro-services/data-export/source/data_export/export/google_cloud_parallel_exporter.py new file mode 100644 index 00000000..65a86574 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/google_cloud_parallel_exporter.py @@ -0,0 +1,30 @@ + +from data_export.google.google_cloud_work_queue import GoogleCloudWorkQueue +import json + +class GoogleCloudParallelExporter: + def __init__(self, config): + self.config = config + + self.work_queue = GoogleCloudWorkQueue(config) + + def export(self, output_dataset_path, dataset): + print("Exporting dataset", dataset) + + task_count = int(self.config["exporter"]["task_count"]) + + for task_id in range(task_count): + self.work_queue.push(json.dumps({ + "task_id" : task_id, + "task_count" : task_count, + "dataset" : dataset})) + + self.start_job() + + def start_job(self): + kubernetes_directory = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "build-scripts", "kubernetes") + command = "kubectl apply -f run-export-worker.yaml" + subprocess.run(command, cwd = kubernetes_directory) + + + diff --git a/platform/micro-services/data-export/source/data_export/export/local_exporter.py b/platform/micro-services/data-export/source/data_export/export/local_exporter.py new file mode 100644 index 00000000..8a61b29e --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/local_exporter.py @@ -0,0 +1,13 @@ + +from data_export.utility.load_dataset_csv import load_dataset_csv +from data_export.utility.write_samples_to_tar_gz import write_samples_to_tar_gz + +class LocalExporter: + def __init__(self, config): + self.config = config + + def export(self, path, dataset): + samples = load_dataset_csv(dataset, task_id=0, task_count=1) + + write_samples_to_tar_gz(samples, path) + diff --git a/platform/micro-services/data-export/source/data_export/google/export_worker.py b/platform/micro-services/data-export/source/data_export/google/export_worker.py new file mode 100644 index 00000000..253fd51f --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/google/export_worker.py @@ -0,0 +1,41 @@ + +from data_export.google.google_cloud_work_queue import GoogleCloudWorkQueue +from data_export.utility.get_config import get_config +from data_export.utility.load_dataset_csv import load_dataset_csv +from data_export.utility.write_samples_to_tar_gz import write_samples_to_tar_gz + +import json + +def main(): + config = get_config() + q = GoogleCloudWorkQueue(config) + print("Worker with sessionID: " + q.sessionID()) + print("Initial queue state: empty=" + str(q.empty())) + while not q.empty(): + item = q.lease(lease_secs=3600, block=True, timeout=2) + if item is not None: + itemstr = item.decode("utf-8") + print("Working on " + itemstr) + export_data(itemstr, config) + q.complete(item) + else: + print("Waiting for work") + print("Queue empty, exiting") + +def export_data(itemstr, config): + item = json.loads(itemstr) + + task_id = item["task_id"] + task_count = item["task_count"] + dataset = item["dataset"] + + samples = load_dataset_csv(dataset, task_id, task_count) + + path = config["exporter"]["output_path"] + "-" + str(task_id) + "-tar.gz" + + write_samples_to_tar_gz(samples, path) + +if __name__ == "__main__": + main() + + diff --git a/platform/micro-services/data-export/source/data_export/google/google_cloud_work_queue.py b/platform/micro-services/data-export/source/data_export/google/google_cloud_work_queue.py new file mode 100644 index 00000000..0b38a3b4 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/google/google_cloud_work_queue.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +# Based on http://peter-hoffmann.com/2012/python-simple-queue-redis-queue.html +# and the suggestion in the redis documentation for RPOPLPUSH, at +# http://redis.io/commands/rpoplpush, which suggests how to implement a work-queue. + + +import redis +import uuid +import hashlib + +from data_export.google.redis_service import RedisService + +class GoogleCloudWorkQueue(object): + """Simple Finite Work Queue with Redis Backend + + This work queue is finite: as long as no more work is added + after workers start, the workers can detect when the queue + is completely empty. + + The items in the work queue are assumed to have unique values. + + This object is not intended to be used by multiple threads + concurrently. + """ + def __init__(self, config): + """The default connection parameters are: host='localhost', port=6379, db=0 + + The work queue is identified by "name". The library may create other + keys with "name" as a prefix. + """ + + self.redis_service = RedisService(config) + self._db = redis.StrictRedis(**self.redis_service.kwargs()) + # The session ID will uniquely identify this "worker". + self._session = str(uuid.uuid4()) + # Work queue is implemented as two queues: main, and processing. + # Work is initially in main, and moved to processing when a client picks it up. + name = "exporter-work-queue" + self._main_q_key = name + self._processing_q_key = name + ":processing" + self._lease_key_prefix = name + ":leased_by_session:" + + def push(self, item): + self._db.rpush(item) + + def sessionID(self): + """Return the ID for this session.""" + return self._session + + def _main_qsize(self): + """Return the size of the main queue.""" + return self._db.llen(self._main_q_key) + + def _processing_qsize(self): + """Return the size of the main queue.""" + return self._db.llen(self._processing_q_key) + + def empty(self): + """Return True if the queue is empty, including work being done, False otherwise. + + False does not necessarily mean that there is work available to work on right now, + """ + return self._main_qsize() == 0 and self._processing_qsize() == 0 + +# TODO: implement this +# def check_expired_leases(self): +# """Return to the work queueReturn True if the queue is empty, False otherwise.""" +# # Processing list should not be _too_ long since it is approximately as long +# # as the number of active and recently active workers. +# processing = self._db.lrange(self._processing_q_key, 0, -1) +# for item in processing: +# # If the lease key is not present for an item (it expired or was +# # never created because the client crashed before creating it) +# # then move the item back to the main queue so others can work on it. +# if not self._lease_exists(item): +# TODO: transactionally move the key from processing queue to +# to main queue, while detecting if a new lease is created +# or if either queue is modified. + + def _itemkey(self, item): + """Returns a string that uniquely identifies an item (bytes).""" + return hashlib.sha224(item).hexdigest() + + def _lease_exists(self, item): + """True if a lease on 'item' exists.""" + return self._db.exists(self._lease_key_prefix + self._itemkey(item)) + + def lease(self, lease_secs=60, block=True, timeout=None): + """Begin working on an item the work queue. + + Lease the item for lease_secs. After that time, other + workers may consider this client to have crashed or stalled + and pick up the item instead. + + If optional args block is true and timeout is None (the default), block + if necessary until an item is available.""" + if block: + item = self._db.brpoplpush(self._main_q_key, self._processing_q_key, timeout=timeout) + else: + item = self._db.rpoplpush(self._main_q_key, self._processing_q_key) + if item: + # Record that we (this session id) are working on a key. Expire that + # note after the lease timeout. + # Note: if we crash at this line of the program, then GC will see no lease + # for this item a later return it to the main queue. + itemkey = self._itemkey(item) + self._db.setex(self._lease_key_prefix + itemkey, lease_secs, self._session) + return item + + def complete(self, value): + """Complete working on the item with 'value'. + + If the lease expired, the item may not have completed, and some + other worker may have picked it up. There is no indication + of what happened. + """ + self._db.lrem(self._processing_q_key, 0, value) + # If we crash here, then the GC code will try to move the value, but it will + # not be here, which is fine. So this does not need to be a transaction. + itemkey = self._itemkey(value) + self._db.delete(self._lease_key_prefix + itemkey) + +# TODO: add functions to clean up all keys associated with "name" when +# processing is complete. + +# TODO: add a function to add an item to the queue. Atomically +# check if the queue is empty and if so fail to add the item +# since other workers might think work is done and be in the process +# of exiting. + +# TODO(etune): move to my own github for hosting, e.g. github.com/erictune/rediswq-py and +# make it so it can be pip installed by anyone (see +# http://stackoverflow.com/questions/8247605/configuring-so-that-pip-install-can-work-from-github) + +# TODO(etune): finish code to GC expired leases, and call periodically +# e.g. each time lease times out. + diff --git a/platform/micro-services/data-export/source/data_export/google/redis_service.py b/platform/micro-services/data-export/source/data_export/google/redis_service.py new file mode 100644 index 00000000..d48abb98 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/google/redis_service.py @@ -0,0 +1,51 @@ + +import subprocess +import logging + +logger = logging.getLogger(__name__) + +class RedisService: + def __init__(self, config): + self.config = config + + self.startup() + + def startup(self): + self.instances = self.get_instances() + + if not "exporter-work-queue" in self.instances: + self.start_instance() + + def start_instance(self): + command = ["gcloud", "redis", "instances", "create", "exporter-work-queue", "--size=2", "--region", "europe-west4"] + + subprocess.run(command) + + def get_instances(self): + command = ["gcloud", "redis", "instances", "describe", "exporter-work-queue", "--region", "europe-west4"] + + process = subprocess.run(command, capture_output=True) + + stdout = process.stdout.decode() + + logger.debug("stdout: " + stdout) + + # parse output + instances = {} + + for line in stdout.split('\n'): + logger.debug("parsing line: '" + line + "'") + if line.find("host:") == 0: + host = line[5:].strip() + + if line.find("name:") == 0: + name = line[5:].strip().split("/")[-1] + instances[name] = host + + return instances + + def kwargs(self): + return { "host" : self.instances["exporter-work-queue"], "port" : 6379, "db" : 0 } + + + diff --git a/platform/micro-services/data-export/source/data_export/utility/__init__.py b/platform/micro-services/data-export/source/data_export/utility/__init__.py new file mode 100644 index 00000000..1da8520e --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/utility/__init__.py @@ -0,0 +1,4 @@ + +from data_export.utility.get_config import get_config + + diff --git a/platform/micro-services/data-export/source/data_export/utility/get_config.py b/platform/micro-services/data-export/source/data_export/utility/get_config.py new file mode 100644 index 00000000..375d4af2 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/utility/get_config.py @@ -0,0 +1,47 @@ + +import config +import os +import logging + +def get_config(): + config = setup_config({}) + + setup_logging(config) + + return config + +def setup_config(dictionary): + return config.ConfigurationSet( + config.config_from_env(prefix="MLCOMMONS"), + config.config_from_yaml(config_path(), read_from_file=True), + config.config_from_dict(dictionary), + ) + +def config_path(): + home = os.path.expanduser("~") + home_config_path = os.path.join(home, ".mlcommons", "config.yaml") + if os.path.exists(home_config_path): + return home_config_path + + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "configs", "server.yaml") + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + diff --git a/platform/micro-services/data-export/source/data_export/utility/load_dataset_csv.py b/platform/micro-services/data-export/source/data_export/utility/load_dataset_csv.py new file mode 100644 index 00000000..3d4e0856 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/utility/load_dataset_csv.py @@ -0,0 +1,31 @@ + +from smart_open import open + +import csv + +import json + +import logging + +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("google").setLevel(logging.WARNING) + +def load_dataset_csv(csv_path, task_id, task_count): + + with open(csv_path) as csv_file: + return (yield from load_dataset_csv_from_file(csv_file, task_id, task_count)) + +def load_dataset_csv_from_file(csv_file, task_id, task_count): + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + index = 0 + for row in reader: + path, caption = row[0], row[1] + + metadata = {} + if len(row) >= 3: + metadata = json.loads(row[2]) + + if index % task_count == task_id: + yield {"path" : path, "caption" : caption, "metadata" : metadata} + + index += 1 diff --git a/platform/micro-services/data-export/source/data_export/utility/write_samples_to_tar_gz.py b/platform/micro-services/data-export/source/data_export/utility/write_samples_to_tar_gz.py new file mode 100644 index 00000000..b407ade6 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/utility/write_samples_to_tar_gz.py @@ -0,0 +1,51 @@ + +import os +import shutil +import subprocess +import csv +import json +import errno + +def write_samples_to_tar_gz(samples, path): + os.makedirs("/tmp/export-data", exist_ok=True) + + # copy locally + command = ["gsutil", "-m", "cp", "-I", "/tmp/export-data"] + + stream_to_command(command, samples) + + # tar it + filename = os.path.basename(path) + + temp_archive = os.path.join("/tmp", filename) + + command = ["tar", "-czvf", temp_archive, "export-data"] + + subprocess.run(command, cwd="/tmp") + + # upload it + command = ["gsutil", "-m", "cp", temp_archive, path] + + subprocess.run(command) + +def stream_to_command(command, samples): + p = subprocess.Popen(command, stdin=subprocess.PIPE) + with open("/tmp/export-data/dataset.csv", "w", newline="") as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"') + + for sample in samples: + writer.writerow([sample["path"], sample["caption"], json.dumps(sample["metadata"])]) + line = sample["path"] + "\n" + try: + p.stdin.write(line.encode('utf-8')) + except IOError as e: + if e.errno == errno.EPIPE or e.errno == errno.EINVAL: + # Stop loop on "Invalid pipe" or "Invalid argument". + # No sense in continuing with broken pipe. + break + else: + # Raise any other error. + raise + + p.stdin.close() + p.wait() diff --git a/platform/micro-services/data-export/source/flask/app.py b/platform/micro-services/data-export/source/flask/app.py new file mode 100644 index 00000000..289198fb --- /dev/null +++ b/platform/micro-services/data-export/source/flask/app.py @@ -0,0 +1,18 @@ +from flask import Flask, request +from flask_cors import CORS #comment this on deployment + +import data_export + +import logging + +app = Flask(__name__) +CORS(app) #comment this on deployment + +logger = logging.getLogger(__name__) + +@app.route('/peoples_speech/export_dataset', methods=['GET', 'POST']) +def export_dataset(): + dataset = request.json["dataset"] + output_dataset = request.json["output_dataset"] + data_export.export_dataset(output_dataset, dataset) + return { "results_path" : output_dataset } diff --git a/platform/micro-services/data-export/start-dev b/platform/micro-services/data-export/start-dev new file mode 100755 index 00000000..31eabc6b --- /dev/null +++ b/platform/micro-services/data-export/start-dev @@ -0,0 +1,51 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH="$LOCAL_DIRECTORY/source/flask" +PYTHONPATH+=":$LOCAL_DIRECTORY/source" +export PYTHONPATH + +# export flask +export FLASK_APP=$LOCAL_DIRECTORY/source/flask/app.py +export FLASK_DEBUG=1 + +# kill background tasks on script exit +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +# Start the dev environment +python -m flask run & +wait + + diff --git a/platform/micro-services/data-export/start-production b/platform/micro-services/data-export/start-production new file mode 100755 index 00000000..1c653dd0 --- /dev/null +++ b/platform/micro-services/data-export/start-production @@ -0,0 +1,51 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH="$LOCAL_DIRECTORY/source/flask" +PYTHONPATH+=":$LOCAL_DIRECTORY/source" +export PYTHONPATH + +# export flask +export FLASK_APP=$LOCAL_DIRECTORY/source/flask/app.py +export FLASK_DEBUG=0 + +# kill background tasks on script exit +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +# Start the dev environment +python -m flask run --host=0.0.0.0 --port=$1 & +wait + + diff --git a/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py b/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py index ffea867e..ffdb4f4b 100644 --- a/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py +++ b/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py @@ -1,3 +1,4 @@ from data_export.api.save_dataset import save_dataset +from data_export.api.setup_cli_parser import setup_cli_parser diff --git a/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.sh b/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.sh new file mode 100755 index 00000000..adab36dc --- /dev/null +++ b/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.sh @@ -0,0 +1 @@ +gcloud builds submit --config cloudbuild.yaml ../.. diff --git a/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.yaml b/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.yaml new file mode 100644 index 00000000..00ca2d40 --- /dev/null +++ b/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.yaml @@ -0,0 +1,3 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: [ 'build', '-t', 'gcr.io/peoples-speech/data-export', '-f', 'build-scripts/docker/Dockerfile', '.' ] diff --git a/platform/micro-services/website/build-scripts/docker/Dockerfile b/platform/micro-services/website/build-scripts/docker/Dockerfile new file mode 100644 index 00000000..76fcae76 --- /dev/null +++ b/platform/micro-services/website/build-scripts/docker/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:20.04 + +COPY . /app + +RUN apt-get update && apt-get install -y python3-pip && pip3 install -r /app/requirements.txt + +CMD /app/start-production + diff --git a/platform/micro-services/website/build-scripts/docker/build-container.sh b/platform/micro-services/website/build-scripts/docker/build-container.sh new file mode 100755 index 00000000..561a0400 --- /dev/null +++ b/platform/micro-services/website/build-scripts/docker/build-container.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +docker build -t peoples-speech-platform:0.1 -f $LOCAL_DIRECTORY/Dockerfile $LOCAL_DIRECTORY/../.. + diff --git a/platform/micro-services/website/build-scripts/kubernetes/deploy-staging.sh b/platform/micro-services/website/build-scripts/kubernetes/deploy-staging.sh new file mode 100644 index 00000000..0254b043 --- /dev/null +++ b/platform/micro-services/website/build-scripts/kubernetes/deploy-staging.sh @@ -0,0 +1,2 @@ +gcloud container clusters get-credentials peoples-speech-platform +kubectl create deployment website --image=gcr.io/peoples-speech/platform:latest diff --git a/platform/micro-services/website/start-production b/platform/micro-services/website/start-production new file mode 100755 index 00000000..cea26de8 --- /dev/null +++ b/platform/micro-services/website/start-production @@ -0,0 +1,66 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH="$LOCAL_DIRECTORY/source/flask" +export PYTHONPATH + +# Setup google credentials +export GOOGLE_APPLICATION_CREDENTIALS=$LOCAL_DIRECTORY/source/config/gcloud_key.json + +# export flask +export FLASK_APP=$LOCAL_DIRECTORY/source/flask/app.py + +# Setup the react dev environment +REACT_DEV_ENVIRONMENT=$LOCAL_DIRECTORY/react-development + +# Setup the dev environment if it doesn't exist +if [ ! -d $REACT_DEV_ENVIRONMENT ]; then +npx create-react-app $REACT_DEV_ENVIRONMENT +fi + +# Move the code over +rsync -av --delete $LOCAL_DIRECTORY/source/react/ $REACT_DEV_ENVIRONMENT/src/ +rsync -av --delete $LOCAL_DIRECTORY/source/config/ $REACT_DEV_ENVIRONMENT/src/config/ + +# kill background tasks on script exit +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +# Start the dev environment +cd $REACT_DEV_ENVIRONMENT +python -m flask run & +COLOR=1 npm start | cat & +wait + + diff --git a/platform/peoples-speech b/platform/peoples-speech index 437320a7..af1105e3 100755 --- a/platform/peoples-speech +++ b/platform/peoples-speech @@ -16,11 +16,11 @@ if [[ $PYTHON_ENV == 0 ]]; then echo "Not in virtual environment" -ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate +ACTIVATE=/tmp/mlcommons-environment/bin/activate if [ ! -f $ACTIVATE ]; then echo "Virtual environment doesn't exist, making it..." -python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m venv /tmp/mlcommons-environment python -m pip install --upgrade pip > /dev/null fi diff --git a/platform/requirements.txt b/platform/requirements.txt index fd3a245b..42020b04 100644 --- a/platform/requirements.txt +++ b/platform/requirements.txt @@ -4,3 +4,4 @@ gtts python-configuration[yaml] whoosh audiofile +redis From 842b7b8c3dc28002d2f577d3c84218ad5cc20d85 Mon Sep 17 00:00:00 2001 From: greg-landing Date: Wed, 7 Jul 2021 13:57:57 -0700 Subject: [PATCH 3/3] ENH: Update --- .../source/cli/peoples-speech.py | 8 +- .../data_export/api/setup_cli_parser.py | 10 -- .../quality/scripts/sample-dataset.py | 131 ++++++++++++++++++ 3 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 platform/micro-services/quality/scripts/sample-dataset.py diff --git a/platform/micro-services/command-line-interface/source/cli/peoples-speech.py b/platform/micro-services/command-line-interface/source/cli/peoples-speech.py index d0140e3d..f67d34db 100644 --- a/platform/micro-services/command-line-interface/source/cli/peoples-speech.py +++ b/platform/micro-services/command-line-interface/source/cli/peoples-speech.py @@ -3,14 +3,14 @@ from argparse import ArgumentParser -import peoples_speech.data_book -import peoples_speech.data_export -import peoples_speech.task_manager - import config import logging +import peoples_speech.data_book +import peoples_speech.data_export +import peoples_speech.task_manager + logger = logging.getLogger(__name__) def main(): diff --git a/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py b/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py index dcb46ef9..d5a9023f 100644 --- a/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py +++ b/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py @@ -11,15 +11,9 @@ def setup_cli_parser(subparsers): parser = subparsers.add_parser('export') -<<<<<<< HEAD parser.add_argument("-i", "--input-dataset", default="gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.csv", help="The dataset to export.") parser.add_argument("-o", "--output-dataset-path", default="gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.tar.gz", help="The path to save the new dataset.") parser.add_argument("-c", "--config-file-path", default="", help="The path to the config file.") -======= - parser.add_argument("-i", "--input-dataset", default="gs://the-peoples-speech-west-europe/peoples-speech-v0.5/train.csv", help="The dataset to export.") - parser.add_argument("-o", "--output-dataset-path", default="", help="The path to save the new dataset.") - parser.add_argument("-c", "--config-file-path", default=".csv", help="The path to the config file.") ->>>>>>> d17bf137... TEST: Added unit test for data export. parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") @@ -34,11 +28,7 @@ def dispatch(args): logger.debug("Full config: " + str(config)) -<<<<<<< HEAD export_dataset(config["output_dataset_path"], config["input_dataset"], config) -======= - export_dataset(config["input_dataset"], config) ->>>>>>> d17bf137... TEST: Added unit test for data export. def setup_config(dictionary): return config.ConfigurationSet( diff --git a/platform/micro-services/quality/scripts/sample-dataset.py b/platform/micro-services/quality/scripts/sample-dataset.py new file mode 100644 index 00000000..4444b722 --- /dev/null +++ b/platform/micro-services/quality/scripts/sample-dataset.py @@ -0,0 +1,131 @@ + +import os + +from argparse import ArgumentParser + +import config +import jsonlines +import random +from smart_open import open + +import logging + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("Randomly sample audio and transcripts from a json lines file.") + + subparsers = parser.add_subparsers() + + setup_cli_parser(subparsers) + + args = parser.parse_args() + + args.func(args) + +def setup_cli_parser(subparsers): + + parser = subparsers.add_parser('dataset') + + parser.add_argument("-i", "--dataset-path", + default="gs://the-peoples-speech-west-europe/forced-aligner/cuda-forced-aligner/output_work_dir_5b/output_work_dir_5b/dataset_manifest_mp3_956_all.json", + help="Path to json lines file describing the dataset.") + parser.add_argument("-o", "--output-dataset-path", default="data/dataset.json", help="The path to save the new dataset.") + parser.add_argument("-c", "--config-file-path", default="", help="The path to the config file.") + parser.add_argument("-m", "--maximum-samples", default=100, help="How many samples to download.") + parser.add_argument("--maximum-dataset-size", default=1000, help="How many samples to scan.") + parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") + parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") + + parser.set_defaults(func=dispatch) + +def dispatch(args): + arguments = vars(args) + + config = setup_config(arguments) + + setup_logging(config) + + logger.debug("Full config: " + str(config)) + + samples = load_samples(config) + + filtered_samples = filter_samples(samples, config) + + save_samples(filtered_samples, config) + +def load_samples(config): + samples = [] + + with open(config["dataset_path"]) as dataset_file: + + dataset_reader = jsonlines.Reader(dataset_file) + + for line in dataset_reader: + for path, label in zip(line["training_data"]["labels"], line["training_data"]["output_paths"]): + samples.append(path, label) + + if len(samples) > int(config["maximum_dataset_size"]): + break + + return samples + +def filter_samples(samples, config): + random.seed(42) + random.shuffle(samples) + + limit = min(len(samples), config["maximum_samples"]) + + return samples[:limit] + +def save_samples(samples, config): + + output_directory = os.path.dirname(config["output_dataset_path"]) + os.mkdirs(output_directory, exist_ok=True) + + with open(config["output_dataset_path"]) as dataset_file: + dataset_writer = jsonlines.Writer(dataset_file) + + for path, label in samples: + local_path = download(path, output_directory) + dataset_writer({"training_datset" : { "output_paths": [local_path], "labels" : [label] }}) + +def setup_config(dictionary): + return config.ConfigurationSet( + config.config_from_env(prefix="MLCOMMONS"), + config.config_from_yaml(config_path(), read_from_file=True), + config.config_from_dict(dictionary), + ) + +def config_path(): + home = os.path.expanduser("~") + home_config_path = os.path.join(home, ".mlcommons", "config.yaml") + if os.path.exists(home_config_path): + return home_config_path + + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "config", "default.yaml") + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + +if __name__ == "__main__": + main() + +