Add re-flagging based on multiple filters

pyproject.toml

@@ -13,7 +13,7 @@ classifiers = []
 [tool.poetry.dependencies]
 python = ">=3.9, <3.14"
 pandas = ">=2.1.4,<2.2"
-click = "^7.1.2"
+click = "^8.1"
 pyarrow = ">=14.0.2"
 terminaltables = "^3.1.0"
 urllib3 = "^1.26.10"

quantaq_cli/console/__init__.py

@@ -1,6 +1,6 @@
 import pkg_resources
 import rich_click as click
-from quantaq_cli.variables import SUPPORTED_MODELS
+from quantaq_cli.variables import SUPPORTED_MODELS, SUPPORTED_SOURCES
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
@@ -52,7 +52,13 @@ def merge(files, tscol, output, verbose, **kwargs):
 @click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
 @click.option("--on", default="timestamp", help="Name of the datetime column to resample over.", type=str)
 @click.option("--by", default=None, help="Optional column(s) to group by first")
-@click.option("--wind", default=("wx_u", "wx_v", "wx_ws", "wx_wd"), help="``(u, v, speed, direction)`` column names")
+@click.option(
+    "--wind",
+    nargs=4,
+    type=str,
+    default=("wx_u", "wx_v", "wx_ws", "wx_wd"),
+    help="(u, v, speed, direction) column names",
+)
 @click.option("--numeric_how", default="mean", help="Aggregation for numeric columns.")
 @click.option("--nonnumeric_how", default="first", help="Aggregation for non-numeric columns.")
 @click.option("-v", "--verbose", is_flag=True, help="Enable verbose mode (debugging)")
@@ -69,38 +75,24 @@ def resample(file, rule, output, verbose, **kwargs):
 @click.option("-d", "--dry-run", is_flag=True, help="Print table to screen and bypass file save")
 @click.option("-t", "--table", is_flag=True, help="Print as a dict instead of as a table")
 @click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
-@click.option("-f", "--flag", default="flag", help="The name of the flag column", type=str)
 @click.option("-v", "--verbose", is_flag=True, help="Enable verbose mode (debugging)")
-@click.option("-m", "--model", default="modulair_pm", help="The device model type. One of {}".format(SUPPORTED_MODELS))
-def expunge(file, dry_run, table, output, flag, verbose, model, **kwargs):
+def expunge(file, dry_run, table, output, verbose):
     """Expunge (NaN flagged values) FILE and save to OUTPUT.
     """
     from .commands.expunge import expunge_command
 
-    expunge_command(file, output, flagcol=flag, dry_run=dry_run, verbose=verbose, model=model, table=table, **kwargs)
+    expunge_command(file, output, dry_run=dry_run, verbose=verbose, table=table)
 
 
 @click.command("flag", short_help="flag data based on specific criteria")
 @click.argument("file", nargs=1, type=click.Path())
-@click.argument("column", nargs=1, type=str)
-@click.argument("comparator", nargs=1, type=str)
-@click.argument("value", nargs=1, type=float)
-@click.option("-f", "--flag", default="FLAG_ROW", help="One of [FLAG_OPC, FLAG_CO, FLAG_NO, FLAG_NO2, FLAG_O3, FLAG_CO2, FLAG_ROW]")
 @click.option("-o", "--output", default="output.csv", help="The filepath where you would like to save the file", type=str)
 @click.option("-v", "--verbose", is_flag=True, help="Enable verbose mode (debugging)")
-@click.option("-m", "--model", default="modulair_pm", help="The device model type. One of {}".format(SUPPORTED_MODELS))
-def flag(file, column, comparator, value, flag, output, verbose, model, **kwargs):
-    """Set a FLAG based on user input or statistical method.
-
-    Four arguments are required:
-      1. FILE -> the path to the file of interest
-      2. COLUMN -> the exact name of the column
-      3. COMPARATOR -> one of ['lt', 'gt', 'eq', 'le', 'ge']
-      4. VALUE -> the value by which to filter
-    """
+def flag(file, output, verbose):
+    """Reflag a data file"""
     from .commands.flag import flag_command
 
-    flag_command(file, column, comparator, value, output, flag=flag, verbose=verbose, model=model)
+    flag_command(file, output, verbose=verbose)
 
 
 @click.command("clean")

quantaq_cli/console/commands/expunge.py

@@ -1,30 +1,56 @@
 from pathlib import Path
-import pandas as pd
-import numpy as np
+
 import click
-from terminaltables import SingleTable
+import numpy as np
+import pandas as pd
+
+from quantaq_cli.exceptions import InvalidFileExtension, InvalidDeviceModel
+from quantaq_cli.utilities import safe_load, determine_timestamp_column
+from quantaq_cli.variables import FLAG_DEFINITIONS, SUPPORTED_MODELS
+from quantaq_cli.console.commands.flag import (
+    echo_flag_table
+)
 
-from ...exceptions import InvalidFileExtension, InvalidDeviceModel
-from ...utilities import safe_load
-from ...variables import FLAGS, SUPPORTED_MODELS
 
+def expunge_dataframe(df):
+
+    # get the flags (in the future, this will come from the file itself)
+    list_of_flags = FLAG_DEFINITIONS
+
+    # force the flag column to be an int
+    df["flag"] = df["flag"].astype(int, errors='ignore')
+
+    # Drop NaNs
+    df = df.dropna(how='any', subset=["flag"])
+
+    for label, value, cols in list_of_flags:
+        mask = df["flag"] & value == value
+        if not mask.any():
+            continue
+
+        # NaN the necessary columns
+        if cols == "all_columns":
+            tscol = determine_timestamp_column(df)
+            cols_to_keep = {tscol, "sn", "flag"}
+            cols = [c for c in df.columns if c not in cols_to_keep]
+        elif len(cols) > 0:
+            cols = [c for c in cols if c in df.columns]
+
+        # set the mask
+        df.loc[mask, cols] = np.nan
+
+    return df
 
 def expunge_command(file, output, **kwargs):
     verbose = kwargs.pop("verbose", False)
     dry_run = kwargs.pop("dry_run", False)
-    flagcol = kwargs.pop("flagcol", "flag")
-    model   = kwargs.pop("model", "modulair_pm")
     table   = kwargs.pop("table", False)
 
     # make sure the extension is either a csv or feather format
     output = Path(output)
     if output.suffix not in (".csv", ".feather"):
         raise InvalidFileExtension("Invalid file extension")
 
-    # ensure the model is valid
-    if model not in SUPPORTED_MODELS:
-        raise InvalidDeviceModel("Invalid device model. Must be one of {}".format(SUPPORTED_MODELS))
-
     save_as_csv = True if output.suffix == ".csv" else False
 
     # concat everything in filepath
@@ -34,49 +60,15 @@ def expunge_command(file, output, **kwargs):
     # load the file
     df = safe_load(file)
 
+    # expunge
     if verbose:
-        click.echo("Expunging data for {}".format(model))
-
-    # init an array to hold the table data
-    data = []
-    data.append(["FLAG", "FLAG VALUE", "# OCCURENCES", "% DATA"])
+        click.echo("Expunging data for {}".format(file))
 
-    # get the flags (in the future, this will come from the file itself)
-    list_of_flags = FLAGS.get(model)
-
-    # force the flag column to be an int
-    df[flagcol] = df[flagcol].astype(int, errors='ignore')
-
-    # Drop NaNs
-    df = df.dropna(how='any', subset=[flagcol])
-
-    for label, value, cols in list_of_flags:
-        mask = df[flagcol] & value == value
-        n_affected = mask.sum()
-        pct_affected = round((n_affected / df.shape[0]) * 100.0, 2)
-
-        # NaN the necessary columns
-        if cols is None:
-            cols = df.columns
-        elif len(cols) > 0:
-            cols = [c for c in cols if c in df.columns]
-
-        # set the mask
-        df.loc[mask, cols] = np.nan
-
-        # add a row to the output table
-        data.append([label, value, n_affected, pct_affected])
+    df = expunge_dataframe(df)
 
     if dry_run or verbose:
-        tbl = SingleTable(data)
-        tbl.title = "Flag Breakdown".upper()
-
-        if table:
-            click.echo(tbl.table)
-        else:
-            for item in data:
-                click.echo(item)
-
+        echo_flag_table(df)
+
     # save the file (if not a dry run)
     if not dry_run:
         if verbose: