Refactor to library-first architecture with thin CLI wrapper

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "quantaq-cli"
 version = "1.0.0rc0"
-description = "The QuantAQ CLI"
+description = "The QuantAQ Python Toolkit and CLI"
 authors = ["David H Hagan <david.hagan@quant-aq.com>"]
 license = "Apache-2.0"
 readme = "README.md"
@@ -13,13 +13,15 @@ classifiers = []
 [tool.poetry.dependencies]
 python = ">=3.9, <3.14"
 pandas = ">=2.1.4,<2.2"
-click = "^7.1.2"
+click = "^8.1"
 pyarrow = ">=14.0.2"
 terminaltables = "^3.1.0"
 urllib3 = "^1.26.10"
 numpy = "^1.23.2"
 rich-click = ">=1.7.2"
 loguru = "^0.7.3"
+jupyter = "^1.1.1"
+pandera = ">=0.24,<0.29"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^6.2.5"
@@ -32,7 +34,7 @@ sphinx-rtd-theme = "^2.0"
 setuptools = "^65.6.3"
 
 [tool.poetry.scripts]
-quantaq-cli = "quantaq_cli.console:main"
+quantaq-cli = "quantaq_cli.cli:main"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

quantaq_cli/__init__.py

@@ -0,0 +1,18 @@
+from quantaq_cli.toolkit.concat import concat_files
+from quantaq_cli.toolkit.merge import merge_files
+from quantaq_cli.toolkit.resample import resample_dataframe
+from quantaq_cli.toolkit.flag import flag_dataframe, echo_flag_table, flag_summary
+from quantaq_cli.toolkit.expunge import expunge_dataframe
+from quantaq_cli.toolkit.munge import clean_dataframe
+
+
+__all__ = [
+    "concat_files",
+    "merge_files",
+    "resample_dataframe",
+    "flag_dataframe",
+    "echo_flag_table",
+    "expunge_dataframe",
+    "flag_summary",
+    "clean_dataframe"
+]

quantaq_cli/cli.py

@@ -0,0 +1,242 @@
+from pathlib import Path
+
+import pkg_resources
+import rich_click as click
+from loguru import logger
+
+from quantaq_cli import concat_files, merge_files, resample_dataframe, clean_dataframe
+from quantaq_cli import flag_dataframe, echo_flag_table, expunge_dataframe
+from quantaq_cli.exceptions import InvalidFileExtension
+from quantaq_cli.log import configure_logging, LOG_LEVELS
+from quantaq_cli.toolkit.resample import WIND_COLUMNS
+from quantaq_cli.utilities import safe_load
+from quantaq_cli.variables import SUPPORTED_MODELS, SUPPORTED_SOURCES
+
+
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
+
+__version__ = pkg_resources.get_distribution('quantaq_cli').version
+
+
+@click.group(context_settings=CONTEXT_SETTINGS)
+@click.version_option(__version__)
+@click.pass_context
+def main(ctx):
+    pass
+
+
+def save_file(df, output):
+    """Helper func to save a DataFrame to disk as CSV or parquet.
+
+    Args:
+        df (pd.DataFrame): DataFrame to save
+        output (str): Destination filepath.
+    """
+    logger.info("Saving file to {}", output)
+    output = Path(output)
+    if output.suffix == ".csv":
+        df.to_csv(output, index=False)
+    else:
+        df.to_parquet(output, index=False)
+
+def check_extension(output):
+    """Helper func to check if the output extension is either a csv or parquet
+    format.
+
+    Args:
+        output (str): Destination filepath.
+    """
+    output = Path(output)
+    if output.suffix not in (".csv", ".parquet"):
+        error = InvalidFileExtension(f"Invalid file extension; got {output.suffix!r}")
+        logger.error(error)
+        raise error
+    return output
+
+
+@click.command("concat")
+@click.argument("files", nargs=-1, type=click.Path())
+@click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
+@click.option("--log-level", default="INFO",
+              type=click.Choice(LOG_LEVELS, case_sensitive=False),
+              help="loguru log level (default: INFO)")
+def concat_command(files, output, log_level):
+    """Concat FILES and save to OUTPUT."""
+    configure_logging(log_level)
+
+    # make sure the extension is either a csv or parquet format
+    check_extension(output)
+
+    # concat everything in filepath
+    logger.info("Files to read: {}", files)
+
+    df = concat_files(files)
+
+    # save the file
+    save_file(df, output)
+
+
+@click.command("merge", short_help="merge two files together on their timestamp")
+@click.argument("files", nargs=-1, type=click.Path())
+@click.option("-ts", "--tscol", default="timestamp", help="The column by which to join the files", type=str)
+@click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
+@click.option("--log-level", default="INFO",
+              type=click.Choice(LOG_LEVELS, case_sensitive=False),
+              help="loguru log level (default: INFO)")
+def merge_command(files, output, tscol, log_level):
+    """Merge FILES together and save to OUTPUT."""
+    configure_logging(log_level)
+
+    # make sure the extension is either a csv or parquet format
+    check_extension(output)
+
+    # merge everything in filepath
+    logger.info("Files to read: {}", files)
+
+    df = merge_files(files, tscol)
+
+    # save the file
+    save_file(df, output)
+
+
+@click.command("resample", short_help="up/down sample data")
+@click.argument("file", nargs=1, type=click.Path())
+@click.argument("rule", nargs=1, type=str)
+@click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
+@click.option("--on", default="timestamp", help="Name of the datetime column to resample over.", type=str)
+@click.option("--by", default=None, help="Optional column(s) to group by first")
+@click.option(
+    "--wind",
+    nargs=4,
+    type=str,
+    default=("wx_u", "wx_v", "wx_ws", "wx_wd"),
+    help="(u, v, speed, direction) column names. The u/v columns don't need to be present in the dataframe, but the column names need to be specified.",
+)
+@click.option("--numeric_how", default="mean", help="Aggregation for numeric columns.")
+@click.option("--nonnumeric_how", default="first", help="Aggregation for non-numeric columns.")
+@click.option("--log-level", default="INFO",
+              type=click.Choice(LOG_LEVELS, case_sensitive=False),
+              help="loguru log level (default: INFO)")
+def resample_command(file, rule, output, log_level, **kwargs):
+    """Resample FILE at INTERVAL and save to OUTPUT."""
+    configure_logging(log_level)
+
+    on = kwargs.pop("on", "timestamp")
+    by = kwargs.pop("by", None)
+    wind = kwargs.pop("wind", WIND_COLUMNS)
+    numeric_how = kwargs.pop("numeric_how", "mean")
+    nonnumeric_how = kwargs.pop("nonnumeric_how", "first")
+
+    # make sure the extension is either a csv or parquet format
+    check_extension(output)
+
+    logger.info("File to read: {}", file)
+
+    # load the file
+    df = safe_load(file)
+
+    # if column to resample over needs to be made a datetime obj, do so
+    if on not in df.columns:
+        error = ValueError(f"Invalid column name for the timestamp; got {on!r}")
+        logger.error(error)
+        raise error
+
+    # resample
+    df = resample_dataframe(
+        df,
+        rule,
+        on=on,
+        by=by,
+        wind=wind,
+        numeric_how=numeric_how,
+        nonnumeric_how=nonnumeric_how,
+    )
+
+    # save the file
+    save_file(df, output)
+
+
+@click.command("flag")
+@click.argument("file", nargs=1, type=click.Path())
+@click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
+@click.option("--log-level", default="INFO",
+              type=click.Choice(LOG_LEVELS, case_sensitive=False),
+              help="loguru log level (default: INFO)")
+def flag_command(file, output, log_level):
+    """Flag FILE and save to OUTPUT."""
+    configure_logging(log_level)
+
+    # make sure the extension is either a csv or parquet format
+    check_extension(output)
+
+    logger.info("File to read: {}", file)
+
+    # load the file
+    df = safe_load(file)
+
+    # flag the dataframe
+    df = flag_dataframe(df)
+
+    # save the file
+    save_file(df, output)
+
+
+@click.command("expunge")
+@click.argument("file", nargs=1, type=click.Path())
+@click.option("-d", "--dry-run", is_flag=True, help="Print table to screen and bypass file save")
+@click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
+@click.option("--log-level", default="INFO",
+              type=click.Choice(LOG_LEVELS, case_sensitive=False),
+              help="loguru log level (default: INFO)")
+def expunge_command(file, output, log_level, dry_run):
+    """Expunge FILE and save to OUTPUT."""
+    configure_logging(log_level)
+
+    # make sure the extension is either a csv or parquet format
+    check_extension(output)
+
+    logger.info("Expunging data for {}", file)
+
+    df = safe_load(file)
+    show_summary = dry_run or log_level == "INFO"
+
+    if show_summary:
+        logger.info("Original flag summary:")
+        echo_flag_table(df)
+
+    df_expunged = expunge_dataframe(df)
+
+    if show_summary:
+        logger.info("New flag summary:")
+        echo_flag_table(df_expunged)
+
+    if not dry_run:
+        save_file(df, output)
+
+
+@click.command("clean")
+@click.argument("file", nargs=1, type=click.Path())
+@click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
+@click.option("--log-level", default="INFO",
+              type=click.Choice(LOG_LEVELS, case_sensitive=False),
+              help="loguru log level (default: INFO)")
+def clean_command(file, output, log_level):
+    """Clean FILE and save to OUTPUT."""
+    configure_logging(log_level)
+
+    # make sure the extension is either a csv or parquet format
+    check_extension(output)
+
+    logger.info("Cleaning data for {}", file)
+    df = safe_load(file)
+    df = clean_dataframe(df)
+    # save the file
+    save_file(df, output)
+
+# add the commands one-by-one
+main.add_command(concat_command)
+main.add_command(merge_command)
+main.add_command(resample_command)
+main.add_command(expunge_command)
+main.add_command(flag_command)
+main.add_command(clean_command)