diff --git a/platform/micro-services/command-line-interface/source/cli/peoples-speech.py b/platform/micro-services/command-line-interface/source/cli/peoples-speech.py index ab16d058..f67d34db 100644 --- a/platform/micro-services/command-line-interface/source/cli/peoples-speech.py +++ b/platform/micro-services/command-line-interface/source/cli/peoples-speech.py @@ -3,26 +3,42 @@ from argparse import ArgumentParser -import peoples_speech.data_book -import peoples_speech.data_export -import peoples_speech.task_manager - import config import logging +import peoples_speech.data_book +import peoples_speech.data_export +import peoples_speech.task_manager + logger = logging.getLogger(__name__) def main(): parser = ArgumentParser("The MLCommons data engineering framework.") + subparsers = parser.add_subparsers() + + setup_cli_parser(subparsers) + peoples_speech.data_export.setup_cli_parser(subparsers) + + args = parser.parse_args() + + args.func(args) + +def setup_cli_parser(subparsers): + + parser = subparsers.add_parser('dataset') + parser.add_argument("-i", "--data-book-path", default=sample_databook_path(), help="Path to data book to generate a dataset for.") parser.add_argument("-o", "--output-dataset-path", default="", help="The path to save the new dataset.") - parser.add_argument("-c", "--config-file-path", default=".csv", help="The path to save the new dataset.") + parser.add_argument("-c", "--config-file-path", default=".csv", help="The path to the config file.") parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") - arguments = vars(parser.parse_args()) + parser.set_defaults(func=dispatch) + +def dispatch(args): + arguments = vars(args) config = setup_config(arguments) diff --git a/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.sh b/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.sh new file mode 100755 index 00000000..90fb70f4 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.sh @@ -0,0 +1,13 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +gcloud builds submit --config cloudbuild.yaml $LOCAL_DIRECTORY/../../../../../.. diff --git a/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.yaml b/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.yaml new file mode 100644 index 00000000..662f159e --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/cloud-build/cloudbuild.yaml @@ -0,0 +1,14 @@ +steps: +# build the data export container +- name: 'gcr.io/cloud-builders/docker' + args: [ 'build', '-t', 'gcr.io/the-peoples-speech/data-export:0.12', '-f', 'platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile', 'platform/micro-services/data-export' ] +# push container image +- name: "gcr.io/cloud-builders/docker" + args: ["push", "gcr.io/the-peoples-speech/data-export:0.12"] +# deploy container image to GKE staging +- name: "gcr.io/cloud-builders/gke-deploy" + args: + - run + - --filename=platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml + - --location=europe-west4-c + - --cluster=peoples-speech-platform diff --git a/platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile b/platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile new file mode 100644 index 00000000..b8be3999 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/docker/Dockerfile @@ -0,0 +1,9 @@ +FROM python + +COPY . /app + +ENV BOTO_CONFIG=/app/source/configs/google.boto + +EXPOSE 5000 + +CMD /app/start-production $PORT diff --git a/platform/micro-services/data-export/build-scripts/flask/docker/build-container.sh b/platform/micro-services/data-export/build-scripts/flask/docker/build-container.sh new file mode 100755 index 00000000..b75fa3e6 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/docker/build-container.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +docker build -t data-export:0.1 -f $LOCAL_DIRECTORY/Dockerfile $LOCAL_DIRECTORY/../../../../../.. + diff --git a/platform/micro-services/data-export/build-scripts/flask/kubernetes/deploy-staging.sh b/platform/micro-services/data-export/build-scripts/flask/kubernetes/deploy-staging.sh new file mode 100644 index 00000000..78874bb2 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/kubernetes/deploy-staging.sh @@ -0,0 +1,2 @@ +gcloud container clusters get-credentials peoples-speech-platform +kubectl create deployment data-export --image=gcr.io/peoples-speech/data-export:latest diff --git a/platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml b/platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml new file mode 100644 index 00000000..05b5fb7c --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/kubernetes/export-service.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: data-export + name: data-export +spec: + replicas: 1 + selector: + matchLabels: + app: data-export + template: + metadata: + labels: + app: data-export + spec: + containers: + - image: gcr.io/the-peoples-speech/data-export:0.12 + name: data-export + ports: + - containerPort: 5000 + name: tcp-c-5000 + env: + - name: "PORT" + value: "5000" + volumeMounts: + - mountPath: "/app/credentials" + name: gcloud-service-account-key + readOnly: true + volumes: + - name: gcloud-service-account-key + secret: + secretName: gcloud-service-account-key +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: data-export + name: data-export +spec: + selector: + app: data-export + ports: + - port: 5000 + targetPort: 5000 + protocol: TCP + name: tcp-5000 + type: LoadBalancer + diff --git a/platform/micro-services/data-export/build-scripts/flask/kubernetes/run-export-service.yaml b/platform/micro-services/data-export/build-scripts/flask/kubernetes/run-export-service.yaml new file mode 100644 index 00000000..3c0d7a46 --- /dev/null +++ b/platform/micro-services/data-export/build-scripts/flask/kubernetes/run-export-service.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: data-export + name: data-export +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: data-export + template: + metadata: + labels: + app.kubernetes.io/name: data-export + spec: + containers: + - image: gcr.io/peoples-speech/data-export + name: data-export + ports: + - containerPort: 8080 diff --git a/platform/micro-services/data-export/requirements.txt b/platform/micro-services/data-export/requirements.txt new file mode 100644 index 00000000..2e86cc7c --- /dev/null +++ b/platform/micro-services/data-export/requirements.txt @@ -0,0 +1,7 @@ +flask +flask_cors +python-configuration[yaml] +smart_open[gcs] +redis +gsutil +google-compute-engine diff --git a/platform/micro-services/data-export/run-tests b/platform/micro-services/data-export/run-tests new file mode 100755 index 00000000..82f0e4a1 --- /dev/null +++ b/platform/micro-services/data-export/run-tests @@ -0,0 +1,41 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH+="$LOCAL_DIRECTORY/source" +export PYTHONPATH + +python source/data_export/api/test/test_export_dataset.py + + diff --git a/platform/micro-services/data-export/source/add-clean-text.py b/platform/micro-services/data-export/source/commands/add-clean-text.py similarity index 100% rename from platform/micro-services/data-export/source/add-clean-text.py rename to platform/micro-services/data-export/source/commands/add-clean-text.py diff --git a/platform/micro-services/data-export/source/add-duration-metadata.py b/platform/micro-services/data-export/source/commands/add-duration-metadata.py similarity index 100% rename from platform/micro-services/data-export/source/add-duration-metadata.py rename to platform/micro-services/data-export/source/commands/add-duration-metadata.py diff --git a/platform/micro-services/data-export/source/add-google-speech-transcript.py b/platform/micro-services/data-export/source/commands/add-google-speech-transcript.py similarity index 100% rename from platform/micro-services/data-export/source/add-google-speech-transcript.py rename to platform/micro-services/data-export/source/commands/add-google-speech-transcript.py diff --git a/platform/micro-services/data-export/source/add-librispeech-metadata.py b/platform/micro-services/data-export/source/commands/add-librispeech-metadata.py similarity index 100% rename from platform/micro-services/data-export/source/add-librispeech-metadata.py rename to platform/micro-services/data-export/source/commands/add-librispeech-metadata.py diff --git a/platform/micro-services/data-export/source/convert-audio-format.py b/platform/micro-services/data-export/source/commands/convert-audio-format.py similarity index 100% rename from platform/micro-services/data-export/source/convert-audio-format.py rename to platform/micro-services/data-export/source/commands/convert-audio-format.py diff --git a/platform/micro-services/data-export/source/convert-cc-search.py b/platform/micro-services/data-export/source/commands/convert-cc-search.py similarity index 100% rename from platform/micro-services/data-export/source/convert-cc-search.py rename to platform/micro-services/data-export/source/commands/convert-cc-search.py diff --git a/platform/micro-services/data-export/source/convert-common-voice.py b/platform/micro-services/data-export/source/commands/convert-common-voice.py similarity index 100% rename from platform/micro-services/data-export/source/convert-common-voice.py rename to platform/micro-services/data-export/source/commands/convert-common-voice.py diff --git a/platform/micro-services/data-export/source/convert-dipco-dataset-format.py b/platform/micro-services/data-export/source/commands/convert-dipco-dataset-format.py similarity index 100% rename from platform/micro-services/data-export/source/convert-dipco-dataset-format.py rename to platform/micro-services/data-export/source/commands/convert-dipco-dataset-format.py diff --git a/platform/micro-services/data-export/source/convert-librispeech.py b/platform/micro-services/data-export/source/commands/convert-librispeech.py similarity index 100% rename from platform/micro-services/data-export/source/convert-librispeech.py rename to platform/micro-services/data-export/source/commands/convert-librispeech.py diff --git a/platform/micro-services/data-export/source/convert-librivox-google-cloud.py b/platform/micro-services/data-export/source/commands/convert-librivox-google-cloud.py similarity index 100% rename from platform/micro-services/data-export/source/convert-librivox-google-cloud.py rename to platform/micro-services/data-export/source/commands/convert-librivox-google-cloud.py diff --git a/platform/micro-services/data-export/source/convert-librivox.py b/platform/micro-services/data-export/source/commands/convert-librivox.py similarity index 100% rename from platform/micro-services/data-export/source/convert-librivox.py rename to platform/micro-services/data-export/source/commands/convert-librivox.py diff --git a/platform/micro-services/data-export/source/convert-timit-dataset-format.py b/platform/micro-services/data-export/source/commands/convert-timit-dataset-format.py similarity index 100% rename from platform/micro-services/data-export/source/convert-timit-dataset-format.py rename to platform/micro-services/data-export/source/commands/convert-timit-dataset-format.py diff --git a/platform/micro-services/data-export/source/convert-voicery.py b/platform/micro-services/data-export/source/commands/convert-voicery.py similarity index 100% rename from platform/micro-services/data-export/source/convert-voicery.py rename to platform/micro-services/data-export/source/commands/convert-voicery.py diff --git a/platform/micro-services/data-export/source/convert-warc-to-csv.py b/platform/micro-services/data-export/source/commands/convert-warc-to-csv.py similarity index 100% rename from platform/micro-services/data-export/source/convert-warc-to-csv.py rename to platform/micro-services/data-export/source/commands/convert-warc-to-csv.py diff --git a/platform/micro-services/data-export/source/count-duration.py b/platform/micro-services/data-export/source/commands/count-duration.py similarity index 100% rename from platform/micro-services/data-export/source/count-duration.py rename to platform/micro-services/data-export/source/commands/count-duration.py diff --git a/platform/micro-services/data-export/source/make-librivox-train-test-split.py b/platform/micro-services/data-export/source/commands/make-librivox-train-test-split.py similarity index 100% rename from platform/micro-services/data-export/source/make-librivox-train-test-split.py rename to platform/micro-services/data-export/source/commands/make-librivox-train-test-split.py diff --git a/platform/micro-services/data-export/source/make-peoples-speech-train-test-splits.py b/platform/micro-services/data-export/source/commands/make-peoples-speech-train-test-splits.py similarity index 100% rename from platform/micro-services/data-export/source/make-peoples-speech-train-test-splits.py rename to platform/micro-services/data-export/source/commands/make-peoples-speech-train-test-splits.py diff --git a/platform/micro-services/data-export/source/make-tar-google-cloud.py b/platform/micro-services/data-export/source/commands/make-tar-google-cloud.py similarity index 100% rename from platform/micro-services/data-export/source/make-tar-google-cloud.py rename to platform/micro-services/data-export/source/commands/make-tar-google-cloud.py diff --git a/platform/micro-services/data-export/source/configs/default.yaml b/platform/micro-services/data-export/source/configs/default.yaml new file mode 100644 index 00000000..2d3b18ac --- /dev/null +++ b/platform/micro-services/data-export/source/configs/default.yaml @@ -0,0 +1,6 @@ +exporter: + type: CloudExporter + endpoint: http://34.91.68.228 + +verbose: True + diff --git a/platform/micro-services/data-export/source/configs/local.yaml b/platform/micro-services/data-export/source/configs/local.yaml new file mode 100644 index 00000000..e5278a66 --- /dev/null +++ b/platform/micro-services/data-export/source/configs/local.yaml @@ -0,0 +1,10 @@ + +datasets: + "0": gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.csv + +exporter: + type: LocalExporter + +output_dataset_path: gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.tar.gz + +verbose: True diff --git a/platform/micro-services/data-export/source/configs/server.yaml b/platform/micro-services/data-export/source/configs/server.yaml new file mode 100644 index 00000000..956e69b2 --- /dev/null +++ b/platform/micro-services/data-export/source/configs/server.yaml @@ -0,0 +1,10 @@ +exporter: + type: GoogleCloudParallelExporter + task_count: 1024 + +google: + work_queue: + name: "export_work_queue" + +verbose: True + diff --git a/platform/micro-services/data-export/source/data_export/__init__.py b/platform/micro-services/data-export/source/data_export/__init__.py index b0596091..9ca01754 100644 --- a/platform/micro-services/data-export/source/data_export/__init__.py +++ b/platform/micro-services/data-export/source/data_export/__init__.py @@ -1,5 +1,7 @@ from data_export.api.save_dataset import save_dataset - +from data_export.api.export_dataset_by_id import export_dataset_by_id +from data_export.api.export_dataset import export_dataset +from data_export.api.setup_cli_parser import setup_cli_parser diff --git a/platform/micro-services/data-export/source/data_export/api/export_dataset.py b/platform/micro-services/data-export/source/data_export/api/export_dataset.py new file mode 100644 index 00000000..387daa3c --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/export_dataset.py @@ -0,0 +1,13 @@ + +from data_export.export import ExporterFactory +from data_export.utility.get_config import get_config + +<<<<<<< HEAD +def export_dataset(output_dataset, dataset, config = get_config()): + exporter = ExporterFactory(config).create() + return exporter.export(output_dataset, dataset) +======= +def export_dataset(dataset, config = get_config()): + exporter = ExporterFactory(config).create() + exporter.export(dataset) +>>>>>>> d17bf137... TEST: Added unit test for data export. diff --git a/platform/micro-services/data-export/source/data_export/api/export_dataset_by_id.py b/platform/micro-services/data-export/source/data_export/api/export_dataset_by_id.py new file mode 100644 index 00000000..8605488f --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/export_dataset_by_id.py @@ -0,0 +1,10 @@ + +from data_export.export import ExporterFactory +from data_export.database import get_dataset_from_id +from data_export.utility.get_config import get_config + +def export_dataset_by_id(dataset_id, config = get_config()): + exporter = ExporterFactory(config).create() + dataset = get_dataset_from_id(config, dataset_id) + exporter.export(dataset) + diff --git a/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py b/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py new file mode 100644 index 00000000..d5a9023f --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/setup_cli_parser.py @@ -0,0 +1,70 @@ +from data_export.api.export_dataset import export_dataset + +import config +import logging +import os + +logger = logging.getLogger(__name__) + + +def setup_cli_parser(subparsers): + + parser = subparsers.add_parser('export') + + parser.add_argument("-i", "--input-dataset", default="gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.csv", help="The dataset to export.") + parser.add_argument("-o", "--output-dataset-path", default="gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.tar.gz", help="The path to save the new dataset.") + parser.add_argument("-c", "--config-file-path", default="", help="The path to the config file.") + parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") + parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") + + parser.set_defaults(func=dispatch) + +def dispatch(args): + arguments = vars(args) + + config = setup_config(arguments) + + setup_logging(config) + + logger.debug("Full config: " + str(config)) + + export_dataset(config["output_dataset_path"], config["input_dataset"], config) + +def setup_config(dictionary): + return config.ConfigurationSet( + config.config_from_env(prefix="MLCOMMONS"), + config.config_from_yaml(config_path(dictionary), read_from_file=True), + config.config_from_dict(dictionary), + ) + +def config_path(dictionary): + if os.path.exists(dictionary["config_file_path"]): + return dictionary["config_file_path"] + + home = os.path.expanduser("~") + home_config_path = os.path.join(home, ".mlcommons", "config.yaml") + if os.path.exists(home_config_path): + return home_config_path + + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "configs", "default.yaml") + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + diff --git a/platform/micro-services/data-export/source/data_export/api/test/test_export_dataset.py b/platform/micro-services/data-export/source/data_export/api/test/test_export_dataset.py new file mode 100644 index 00000000..932a0c21 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/api/test/test_export_dataset.py @@ -0,0 +1,36 @@ +import unittest +import tarfile +import io +from data_export.api.export_dataset import export_dataset +from data_export.utility.load_dataset_csv import load_dataset_csv +from data_export.utility.load_dataset_csv import load_dataset_csv_from_file + +class TestDataExport(unittest.TestCase): + + def test_tiny_local(self): + tiny_dataset = "gs://the-peoples-speech-west-europe/peoples-speech-v0.8/unittest.csv" + tiny_config = { + "exporter" : { "type" : "LocalExporter" } } + + export_dataset("/tmp/export-data/unittest.tar.gz", tiny_dataset, tiny_config) + + self.extract_dataset(tiny_config) + self.verify_dataset(tiny_dataset, tiny_config) + + def extract_dataset(self, config): + with tarfile.open(config["output_dataset_path"], "r:gz") as tar: + members = tar.getmembers() + + def verify_dataset(self, dataset, config): + + samples = load_dataset_csv(dataset, task_id=0, task_count=1) + with tarfile.open(config["output_dataset_path"], "r:gz") as tar: + samples_file = io.TextIOWrapper(tar.extractfile("export-data/dataset.csv")) + loaded_samples = load_dataset_csv_from_file(samples_file, task_id=0, task_count=1) + + for original_sample, new_sample in zip(samples, loaded_samples): + self.assertEqual(original_sample, new_sample) + + +if __name__ == '__main__': + unittest.main() diff --git a/platform/micro-services/data-export/source/data_export/database/__init__.py b/platform/micro-services/data-export/source/data_export/database/__init__.py new file mode 100644 index 00000000..cfeed4f8 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/database/__init__.py @@ -0,0 +1,4 @@ + +from data_export.database.get_dataset_from_id import get_dataset_from_id + + diff --git a/platform/micro-services/data-export/source/data_export/database/get_dataset_from_id.py b/platform/micro-services/data-export/source/data_export/database/get_dataset_from_id.py new file mode 100644 index 00000000..2509e106 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/database/get_dataset_from_id.py @@ -0,0 +1,10 @@ + +def get_dataset_from_id(config, dataset_id): + print(config, config["datasets"].as_dict(), dataset_id) + + if dataset_id in config["datasets"].as_dict(): + return config["datasets"][dataset_id] + + assert False, "Not implemented" + + diff --git a/platform/micro-services/data-export/source/data_export/export/__init__.py b/platform/micro-services/data-export/source/data_export/export/__init__.py new file mode 100644 index 00000000..327771a8 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/__init__.py @@ -0,0 +1,4 @@ + +from data_export.export.exporter_factory import ExporterFactory + + diff --git a/platform/micro-services/data-export/source/data_export/export/cloud_exporter.py b/platform/micro-services/data-export/source/data_export/export/cloud_exporter.py new file mode 100644 index 00000000..9fda1847 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/cloud_exporter.py @@ -0,0 +1,31 @@ + +import requests + +import logging + +logger = logging.getLogger() + +class CloudExporter: + def __init__(self, config): + self.config = config + + def export(self, path, dataset): + url = self.config["exporter"]["endpoint"] + ":" + self.get_port() + "/peoples_speech/export_dataset" + data = { + "dataset" : dataset, + "output_dataset" : path + } + + logger.debug("Submitting POST request to url " + str(url)) + logger.debug(" with data " + str(data)) + response = requests.post(url, json=data) + + if response.status_code == 200: + print("Success") + print(response.json()) + else: + print("Failed to submit with error: " + str(response.status_code)) + + def get_port(self): + return "5000" + diff --git a/platform/micro-services/data-export/source/data_export/export/exporter_factory.py b/platform/micro-services/data-export/source/data_export/export/exporter_factory.py new file mode 100644 index 00000000..fcde7833 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/exporter_factory.py @@ -0,0 +1,20 @@ + +from data_export.export.google_cloud_parallel_exporter import GoogleCloudParallelExporter +from data_export.export.local_exporter import LocalExporter +from data_export.export.cloud_exporter import CloudExporter + +class ExporterFactory: + def __init__(self, config): + self.config = config + + def create(self): + if self.config["exporter"]["type"] == "GoogleCloudParallelExporter": + return GoogleCloudParallelExporter(self.config) + if self.config["exporter"]["type"] == "LocalExporter": + return LocalExporter(self.config) + if self.config["exporter"]["type"] == "CloudExporter": + return CloudExporter(self.config) + + assert False, "Unknown exporter type '" + self.config["exporter"]["type"] + "'" + + diff --git a/platform/micro-services/data-export/source/data_export/export/google_cloud_parallel_exporter.py b/platform/micro-services/data-export/source/data_export/export/google_cloud_parallel_exporter.py new file mode 100644 index 00000000..65a86574 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/google_cloud_parallel_exporter.py @@ -0,0 +1,30 @@ + +from data_export.google.google_cloud_work_queue import GoogleCloudWorkQueue +import json + +class GoogleCloudParallelExporter: + def __init__(self, config): + self.config = config + + self.work_queue = GoogleCloudWorkQueue(config) + + def export(self, output_dataset_path, dataset): + print("Exporting dataset", dataset) + + task_count = int(self.config["exporter"]["task_count"]) + + for task_id in range(task_count): + self.work_queue.push(json.dumps({ + "task_id" : task_id, + "task_count" : task_count, + "dataset" : dataset})) + + self.start_job() + + def start_job(self): + kubernetes_directory = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), "build-scripts", "kubernetes") + command = "kubectl apply -f run-export-worker.yaml" + subprocess.run(command, cwd = kubernetes_directory) + + + diff --git a/platform/micro-services/data-export/source/data_export/export/local_exporter.py b/platform/micro-services/data-export/source/data_export/export/local_exporter.py new file mode 100644 index 00000000..8a61b29e --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/export/local_exporter.py @@ -0,0 +1,13 @@ + +from data_export.utility.load_dataset_csv import load_dataset_csv +from data_export.utility.write_samples_to_tar_gz import write_samples_to_tar_gz + +class LocalExporter: + def __init__(self, config): + self.config = config + + def export(self, path, dataset): + samples = load_dataset_csv(dataset, task_id=0, task_count=1) + + write_samples_to_tar_gz(samples, path) + diff --git a/platform/micro-services/data-export/source/data_export/google/export_worker.py b/platform/micro-services/data-export/source/data_export/google/export_worker.py new file mode 100644 index 00000000..253fd51f --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/google/export_worker.py @@ -0,0 +1,41 @@ + +from data_export.google.google_cloud_work_queue import GoogleCloudWorkQueue +from data_export.utility.get_config import get_config +from data_export.utility.load_dataset_csv import load_dataset_csv +from data_export.utility.write_samples_to_tar_gz import write_samples_to_tar_gz + +import json + +def main(): + config = get_config() + q = GoogleCloudWorkQueue(config) + print("Worker with sessionID: " + q.sessionID()) + print("Initial queue state: empty=" + str(q.empty())) + while not q.empty(): + item = q.lease(lease_secs=3600, block=True, timeout=2) + if item is not None: + itemstr = item.decode("utf-8") + print("Working on " + itemstr) + export_data(itemstr, config) + q.complete(item) + else: + print("Waiting for work") + print("Queue empty, exiting") + +def export_data(itemstr, config): + item = json.loads(itemstr) + + task_id = item["task_id"] + task_count = item["task_count"] + dataset = item["dataset"] + + samples = load_dataset_csv(dataset, task_id, task_count) + + path = config["exporter"]["output_path"] + "-" + str(task_id) + "-tar.gz" + + write_samples_to_tar_gz(samples, path) + +if __name__ == "__main__": + main() + + diff --git a/platform/micro-services/data-export/source/data_export/google/google_cloud_work_queue.py b/platform/micro-services/data-export/source/data_export/google/google_cloud_work_queue.py new file mode 100644 index 00000000..0b38a3b4 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/google/google_cloud_work_queue.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python + +# Based on http://peter-hoffmann.com/2012/python-simple-queue-redis-queue.html +# and the suggestion in the redis documentation for RPOPLPUSH, at +# http://redis.io/commands/rpoplpush, which suggests how to implement a work-queue. + + +import redis +import uuid +import hashlib + +from data_export.google.redis_service import RedisService + +class GoogleCloudWorkQueue(object): + """Simple Finite Work Queue with Redis Backend + + This work queue is finite: as long as no more work is added + after workers start, the workers can detect when the queue + is completely empty. + + The items in the work queue are assumed to have unique values. + + This object is not intended to be used by multiple threads + concurrently. + """ + def __init__(self, config): + """The default connection parameters are: host='localhost', port=6379, db=0 + + The work queue is identified by "name". The library may create other + keys with "name" as a prefix. + """ + + self.redis_service = RedisService(config) + self._db = redis.StrictRedis(**self.redis_service.kwargs()) + # The session ID will uniquely identify this "worker". + self._session = str(uuid.uuid4()) + # Work queue is implemented as two queues: main, and processing. + # Work is initially in main, and moved to processing when a client picks it up. + name = "exporter-work-queue" + self._main_q_key = name + self._processing_q_key = name + ":processing" + self._lease_key_prefix = name + ":leased_by_session:" + + def push(self, item): + self._db.rpush(item) + + def sessionID(self): + """Return the ID for this session.""" + return self._session + + def _main_qsize(self): + """Return the size of the main queue.""" + return self._db.llen(self._main_q_key) + + def _processing_qsize(self): + """Return the size of the main queue.""" + return self._db.llen(self._processing_q_key) + + def empty(self): + """Return True if the queue is empty, including work being done, False otherwise. + + False does not necessarily mean that there is work available to work on right now, + """ + return self._main_qsize() == 0 and self._processing_qsize() == 0 + +# TODO: implement this +# def check_expired_leases(self): +# """Return to the work queueReturn True if the queue is empty, False otherwise.""" +# # Processing list should not be _too_ long since it is approximately as long +# # as the number of active and recently active workers. +# processing = self._db.lrange(self._processing_q_key, 0, -1) +# for item in processing: +# # If the lease key is not present for an item (it expired or was +# # never created because the client crashed before creating it) +# # then move the item back to the main queue so others can work on it. +# if not self._lease_exists(item): +# TODO: transactionally move the key from processing queue to +# to main queue, while detecting if a new lease is created +# or if either queue is modified. + + def _itemkey(self, item): + """Returns a string that uniquely identifies an item (bytes).""" + return hashlib.sha224(item).hexdigest() + + def _lease_exists(self, item): + """True if a lease on 'item' exists.""" + return self._db.exists(self._lease_key_prefix + self._itemkey(item)) + + def lease(self, lease_secs=60, block=True, timeout=None): + """Begin working on an item the work queue. + + Lease the item for lease_secs. After that time, other + workers may consider this client to have crashed or stalled + and pick up the item instead. + + If optional args block is true and timeout is None (the default), block + if necessary until an item is available.""" + if block: + item = self._db.brpoplpush(self._main_q_key, self._processing_q_key, timeout=timeout) + else: + item = self._db.rpoplpush(self._main_q_key, self._processing_q_key) + if item: + # Record that we (this session id) are working on a key. Expire that + # note after the lease timeout. + # Note: if we crash at this line of the program, then GC will see no lease + # for this item a later return it to the main queue. + itemkey = self._itemkey(item) + self._db.setex(self._lease_key_prefix + itemkey, lease_secs, self._session) + return item + + def complete(self, value): + """Complete working on the item with 'value'. + + If the lease expired, the item may not have completed, and some + other worker may have picked it up. There is no indication + of what happened. + """ + self._db.lrem(self._processing_q_key, 0, value) + # If we crash here, then the GC code will try to move the value, but it will + # not be here, which is fine. So this does not need to be a transaction. + itemkey = self._itemkey(value) + self._db.delete(self._lease_key_prefix + itemkey) + +# TODO: add functions to clean up all keys associated with "name" when +# processing is complete. + +# TODO: add a function to add an item to the queue. Atomically +# check if the queue is empty and if so fail to add the item +# since other workers might think work is done and be in the process +# of exiting. + +# TODO(etune): move to my own github for hosting, e.g. github.com/erictune/rediswq-py and +# make it so it can be pip installed by anyone (see +# http://stackoverflow.com/questions/8247605/configuring-so-that-pip-install-can-work-from-github) + +# TODO(etune): finish code to GC expired leases, and call periodically +# e.g. each time lease times out. + diff --git a/platform/micro-services/data-export/source/data_export/google/redis_service.py b/platform/micro-services/data-export/source/data_export/google/redis_service.py new file mode 100644 index 00000000..d48abb98 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/google/redis_service.py @@ -0,0 +1,51 @@ + +import subprocess +import logging + +logger = logging.getLogger(__name__) + +class RedisService: + def __init__(self, config): + self.config = config + + self.startup() + + def startup(self): + self.instances = self.get_instances() + + if not "exporter-work-queue" in self.instances: + self.start_instance() + + def start_instance(self): + command = ["gcloud", "redis", "instances", "create", "exporter-work-queue", "--size=2", "--region", "europe-west4"] + + subprocess.run(command) + + def get_instances(self): + command = ["gcloud", "redis", "instances", "describe", "exporter-work-queue", "--region", "europe-west4"] + + process = subprocess.run(command, capture_output=True) + + stdout = process.stdout.decode() + + logger.debug("stdout: " + stdout) + + # parse output + instances = {} + + for line in stdout.split('\n'): + logger.debug("parsing line: '" + line + "'") + if line.find("host:") == 0: + host = line[5:].strip() + + if line.find("name:") == 0: + name = line[5:].strip().split("/")[-1] + instances[name] = host + + return instances + + def kwargs(self): + return { "host" : self.instances["exporter-work-queue"], "port" : 6379, "db" : 0 } + + + diff --git a/platform/micro-services/data-export/source/data_export/utility/__init__.py b/platform/micro-services/data-export/source/data_export/utility/__init__.py new file mode 100644 index 00000000..1da8520e --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/utility/__init__.py @@ -0,0 +1,4 @@ + +from data_export.utility.get_config import get_config + + diff --git a/platform/micro-services/data-export/source/data_export/utility/get_config.py b/platform/micro-services/data-export/source/data_export/utility/get_config.py new file mode 100644 index 00000000..375d4af2 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/utility/get_config.py @@ -0,0 +1,47 @@ + +import config +import os +import logging + +def get_config(): + config = setup_config({}) + + setup_logging(config) + + return config + +def setup_config(dictionary): + return config.ConfigurationSet( + config.config_from_env(prefix="MLCOMMONS"), + config.config_from_yaml(config_path(), read_from_file=True), + config.config_from_dict(dictionary), + ) + +def config_path(): + home = os.path.expanduser("~") + home_config_path = os.path.join(home, ".mlcommons", "config.yaml") + if os.path.exists(home_config_path): + return home_config_path + + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "configs", "server.yaml") + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + diff --git a/platform/micro-services/data-export/source/data_export/utility/load_dataset_csv.py b/platform/micro-services/data-export/source/data_export/utility/load_dataset_csv.py new file mode 100644 index 00000000..3d4e0856 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/utility/load_dataset_csv.py @@ -0,0 +1,31 @@ + +from smart_open import open + +import csv + +import json + +import logging + +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("google").setLevel(logging.WARNING) + +def load_dataset_csv(csv_path, task_id, task_count): + + with open(csv_path) as csv_file: + return (yield from load_dataset_csv_from_file(csv_file, task_id, task_count)) + +def load_dataset_csv_from_file(csv_file, task_id, task_count): + reader = csv.reader(csv_file, delimiter=',', quotechar='"') + index = 0 + for row in reader: + path, caption = row[0], row[1] + + metadata = {} + if len(row) >= 3: + metadata = json.loads(row[2]) + + if index % task_count == task_id: + yield {"path" : path, "caption" : caption, "metadata" : metadata} + + index += 1 diff --git a/platform/micro-services/data-export/source/data_export/utility/write_samples_to_tar_gz.py b/platform/micro-services/data-export/source/data_export/utility/write_samples_to_tar_gz.py new file mode 100644 index 00000000..b407ade6 --- /dev/null +++ b/platform/micro-services/data-export/source/data_export/utility/write_samples_to_tar_gz.py @@ -0,0 +1,51 @@ + +import os +import shutil +import subprocess +import csv +import json +import errno + +def write_samples_to_tar_gz(samples, path): + os.makedirs("/tmp/export-data", exist_ok=True) + + # copy locally + command = ["gsutil", "-m", "cp", "-I", "/tmp/export-data"] + + stream_to_command(command, samples) + + # tar it + filename = os.path.basename(path) + + temp_archive = os.path.join("/tmp", filename) + + command = ["tar", "-czvf", temp_archive, "export-data"] + + subprocess.run(command, cwd="/tmp") + + # upload it + command = ["gsutil", "-m", "cp", temp_archive, path] + + subprocess.run(command) + +def stream_to_command(command, samples): + p = subprocess.Popen(command, stdin=subprocess.PIPE) + with open("/tmp/export-data/dataset.csv", "w", newline="") as csv_file: + writer = csv.writer(csv_file, delimiter=',', quotechar='"') + + for sample in samples: + writer.writerow([sample["path"], sample["caption"], json.dumps(sample["metadata"])]) + line = sample["path"] + "\n" + try: + p.stdin.write(line.encode('utf-8')) + except IOError as e: + if e.errno == errno.EPIPE or e.errno == errno.EINVAL: + # Stop loop on "Invalid pipe" or "Invalid argument". + # No sense in continuing with broken pipe. + break + else: + # Raise any other error. + raise + + p.stdin.close() + p.wait() diff --git a/platform/micro-services/data-export/source/flask/app.py b/platform/micro-services/data-export/source/flask/app.py new file mode 100644 index 00000000..289198fb --- /dev/null +++ b/platform/micro-services/data-export/source/flask/app.py @@ -0,0 +1,18 @@ +from flask import Flask, request +from flask_cors import CORS #comment this on deployment + +import data_export + +import logging + +app = Flask(__name__) +CORS(app) #comment this on deployment + +logger = logging.getLogger(__name__) + +@app.route('/peoples_speech/export_dataset', methods=['GET', 'POST']) +def export_dataset(): + dataset = request.json["dataset"] + output_dataset = request.json["output_dataset"] + data_export.export_dataset(output_dataset, dataset) + return { "results_path" : output_dataset } diff --git a/platform/micro-services/data-export/start-dev b/platform/micro-services/data-export/start-dev new file mode 100755 index 00000000..31eabc6b --- /dev/null +++ b/platform/micro-services/data-export/start-dev @@ -0,0 +1,51 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH="$LOCAL_DIRECTORY/source/flask" +PYTHONPATH+=":$LOCAL_DIRECTORY/source" +export PYTHONPATH + +# export flask +export FLASK_APP=$LOCAL_DIRECTORY/source/flask/app.py +export FLASK_DEBUG=1 + +# kill background tasks on script exit +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +# Start the dev environment +python -m flask run & +wait + + diff --git a/platform/micro-services/data-export/start-production b/platform/micro-services/data-export/start-production new file mode 100755 index 00000000..1c653dd0 --- /dev/null +++ b/platform/micro-services/data-export/start-production @@ -0,0 +1,51 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH="$LOCAL_DIRECTORY/source/flask" +PYTHONPATH+=":$LOCAL_DIRECTORY/source" +export PYTHONPATH + +# export flask +export FLASK_APP=$LOCAL_DIRECTORY/source/flask/app.py +export FLASK_DEBUG=0 + +# kill background tasks on script exit +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +# Start the dev environment +python -m flask run --host=0.0.0.0 --port=$1 & +wait + + diff --git a/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py b/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py index ffea867e..ffdb4f4b 100644 --- a/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py +++ b/platform/micro-services/peoples-speech/source/peoples_speech/data_export/__init__.py @@ -1,3 +1,4 @@ from data_export.api.save_dataset import save_dataset +from data_export.api.setup_cli_parser import setup_cli_parser diff --git a/platform/micro-services/quality/scripts/sample-dataset.py b/platform/micro-services/quality/scripts/sample-dataset.py new file mode 100644 index 00000000..4444b722 --- /dev/null +++ b/platform/micro-services/quality/scripts/sample-dataset.py @@ -0,0 +1,131 @@ + +import os + +from argparse import ArgumentParser + +import config +import jsonlines +import random +from smart_open import open + +import logging + +logger = logging.getLogger(__name__) + +def main(): + parser = ArgumentParser("Randomly sample audio and transcripts from a json lines file.") + + subparsers = parser.add_subparsers() + + setup_cli_parser(subparsers) + + args = parser.parse_args() + + args.func(args) + +def setup_cli_parser(subparsers): + + parser = subparsers.add_parser('dataset') + + parser.add_argument("-i", "--dataset-path", + default="gs://the-peoples-speech-west-europe/forced-aligner/cuda-forced-aligner/output_work_dir_5b/output_work_dir_5b/dataset_manifest_mp3_956_all.json", + help="Path to json lines file describing the dataset.") + parser.add_argument("-o", "--output-dataset-path", default="data/dataset.json", help="The path to save the new dataset.") + parser.add_argument("-c", "--config-file-path", default="", help="The path to the config file.") + parser.add_argument("-m", "--maximum-samples", default=100, help="How many samples to download.") + parser.add_argument("--maximum-dataset-size", default=1000, help="How many samples to scan.") + parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Print out debug messages.") + parser.add_argument("-vi", "--verbose-info", default=False, action="store_true", help="Print out info messages.") + + parser.set_defaults(func=dispatch) + +def dispatch(args): + arguments = vars(args) + + config = setup_config(arguments) + + setup_logging(config) + + logger.debug("Full config: " + str(config)) + + samples = load_samples(config) + + filtered_samples = filter_samples(samples, config) + + save_samples(filtered_samples, config) + +def load_samples(config): + samples = [] + + with open(config["dataset_path"]) as dataset_file: + + dataset_reader = jsonlines.Reader(dataset_file) + + for line in dataset_reader: + for path, label in zip(line["training_data"]["labels"], line["training_data"]["output_paths"]): + samples.append(path, label) + + if len(samples) > int(config["maximum_dataset_size"]): + break + + return samples + +def filter_samples(samples, config): + random.seed(42) + random.shuffle(samples) + + limit = min(len(samples), config["maximum_samples"]) + + return samples[:limit] + +def save_samples(samples, config): + + output_directory = os.path.dirname(config["output_dataset_path"]) + os.mkdirs(output_directory, exist_ok=True) + + with open(config["output_dataset_path"]) as dataset_file: + dataset_writer = jsonlines.Writer(dataset_file) + + for path, label in samples: + local_path = download(path, output_directory) + dataset_writer({"training_datset" : { "output_paths": [local_path], "labels" : [label] }}) + +def setup_config(dictionary): + return config.ConfigurationSet( + config.config_from_env(prefix="MLCOMMONS"), + config.config_from_yaml(config_path(), read_from_file=True), + config.config_from_dict(dictionary), + ) + +def config_path(): + home = os.path.expanduser("~") + home_config_path = os.path.join(home, ".mlcommons", "config.yaml") + if os.path.exists(home_config_path): + return home_config_path + + return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "config", "default.yaml") + +def setup_logging(arguments): + + logging_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + + if arguments["verbose"]: + logging.basicConfig(level=logging.DEBUG, format=logging_format) + elif arguments["verbose_info"]: + logging.basicConfig(level=logging.INFO, format=logging_format) + else: + logging.basicConfig(level=logging.WARNING, format=logging_format) + + root_logger = logging.getLogger() + + if arguments["verbose"]: + root_logger.setLevel(logging.DEBUG) + elif arguments["verbose_info"]: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + +if __name__ == "__main__": + main() + + diff --git a/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.sh b/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.sh new file mode 100755 index 00000000..adab36dc --- /dev/null +++ b/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.sh @@ -0,0 +1 @@ +gcloud builds submit --config cloudbuild.yaml ../.. diff --git a/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.yaml b/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.yaml new file mode 100644 index 00000000..00ca2d40 --- /dev/null +++ b/platform/micro-services/website/build-scripts/cloud-build/cloudbuild.yaml @@ -0,0 +1,3 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: [ 'build', '-t', 'gcr.io/peoples-speech/data-export', '-f', 'build-scripts/docker/Dockerfile', '.' ] diff --git a/platform/micro-services/website/build-scripts/docker/Dockerfile b/platform/micro-services/website/build-scripts/docker/Dockerfile new file mode 100644 index 00000000..76fcae76 --- /dev/null +++ b/platform/micro-services/website/build-scripts/docker/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:20.04 + +COPY . /app + +RUN apt-get update && apt-get install -y python3-pip && pip3 install -r /app/requirements.txt + +CMD /app/start-production + diff --git a/platform/micro-services/website/build-scripts/docker/build-container.sh b/platform/micro-services/website/build-scripts/docker/build-container.sh new file mode 100755 index 00000000..561a0400 --- /dev/null +++ b/platform/micro-services/website/build-scripts/docker/build-container.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +docker build -t peoples-speech-platform:0.1 -f $LOCAL_DIRECTORY/Dockerfile $LOCAL_DIRECTORY/../.. + diff --git a/platform/micro-services/website/build-scripts/kubernetes/deploy-staging.sh b/platform/micro-services/website/build-scripts/kubernetes/deploy-staging.sh new file mode 100644 index 00000000..0254b043 --- /dev/null +++ b/platform/micro-services/website/build-scripts/kubernetes/deploy-staging.sh @@ -0,0 +1,2 @@ +gcloud container clusters get-credentials peoples-speech-platform +kubectl create deployment website --image=gcr.io/peoples-speech/platform:latest diff --git a/platform/micro-services/website/start-production b/platform/micro-services/website/start-production new file mode 100755 index 00000000..cea26de8 --- /dev/null +++ b/platform/micro-services/website/start-production @@ -0,0 +1,66 @@ +#! /bin/bash + +# Safely execute this bash script +# e exit on first failure +# u unset variables are errors +# f disable globbing on * +# pipefail | produces a failure code if any stage fails +set -euf -o pipefail + +# Get the directory of this script +LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Setup virtual environment +PYTHON_ENV=$(python3 -c "import sys; sys.stdout.write(sys.prefix) if (hasattr(sys, 'real_prefix') or sys.base_prefix != sys.prefix) else sys.stdout.write('0')") +if [[ $PYTHON_ENV == 0 ]]; +then +echo "Not in virtual environment" + +ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate + +if [ ! -f $ACTIVATE ]; then +echo "Virtual environment doesn't exist, making it..." +python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m pip install --upgrade pip > /dev/null +fi + +source $ACTIVATE +else +echo "Running in virtual environment $PYTHON_ENV" +fi + +# Make sure requirements are installed +pip install -r $LOCAL_DIRECTORY/requirements.txt > /dev/null + +# Set python environment +PYTHONPATH="$LOCAL_DIRECTORY/source/flask" +export PYTHONPATH + +# Setup google credentials +export GOOGLE_APPLICATION_CREDENTIALS=$LOCAL_DIRECTORY/source/config/gcloud_key.json + +# export flask +export FLASK_APP=$LOCAL_DIRECTORY/source/flask/app.py + +# Setup the react dev environment +REACT_DEV_ENVIRONMENT=$LOCAL_DIRECTORY/react-development + +# Setup the dev environment if it doesn't exist +if [ ! -d $REACT_DEV_ENVIRONMENT ]; then +npx create-react-app $REACT_DEV_ENVIRONMENT +fi + +# Move the code over +rsync -av --delete $LOCAL_DIRECTORY/source/react/ $REACT_DEV_ENVIRONMENT/src/ +rsync -av --delete $LOCAL_DIRECTORY/source/config/ $REACT_DEV_ENVIRONMENT/src/config/ + +# kill background tasks on script exit +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + +# Start the dev environment +cd $REACT_DEV_ENVIRONMENT +python -m flask run & +COLOR=1 npm start | cat & +wait + + diff --git a/platform/peoples-speech b/platform/peoples-speech index 437320a7..af1105e3 100755 --- a/platform/peoples-speech +++ b/platform/peoples-speech @@ -16,11 +16,11 @@ if [[ $PYTHON_ENV == 0 ]]; then echo "Not in virtual environment" -ACTIVATE=$LOCAL_DIRECTORY/environment/bin/activate +ACTIVATE=/tmp/mlcommons-environment/bin/activate if [ ! -f $ACTIVATE ]; then echo "Virtual environment doesn't exist, making it..." -python3 -m venv $LOCAL_DIRECTORY/environment +python3 -m venv /tmp/mlcommons-environment python -m pip install --upgrade pip > /dev/null fi diff --git a/platform/requirements.txt b/platform/requirements.txt index fd3a245b..42020b04 100644 --- a/platform/requirements.txt +++ b/platform/requirements.txt @@ -4,3 +4,4 @@ gtts python-configuration[yaml] whoosh audiofile +redis