Feature/magic time args (#37)

* implement magic strings

* updated changelog

* put back u-prefixes

* missed u

* more u's

* more u's

* add magic string description to readme

* - refactor duplicate logic in _parse_{min,max}_timestamp into _parse_timestamp
- fix tests to use new DateArgumentException

Author: timabrmsn

CHANGELOG.md

@@ -18,6 +18,7 @@ how a consumer would use the library (e.g. adding unit tests, updating documenta
     - `--filename` flag renamed to `--file-name`.
     - `--filepath` flag renamed to `--file-path`.
     - `--processOwner` flag renamed to `--process-owner`
+- `-b|--begin` and `-e|--end` arguments now accept shorthand date-range strings for days, hours, and minute intervals going back from the current time (e.g. `30d`, `24h`, `15m`).
 - Default profile validation logic added to prevent confusing error states.
 
 ### Added

README.md

@@ -61,14 +61,31 @@ Using the CLI, you can query for events and send them to three possible destinat
 To print events to stdout, do:
 
 ```bash
-code42 security-data print -b 2020-02-02
+code42 security-data print -b <begin_date>
 ```
 
 Note that `-b` or `--begin` is usually required.
-To specify a time, do:
+
+And end date can also be given with `-e` or `--end` to query for a specific date range (if end is not passed, it will get all events up to the present time).
+
+To specify a begin/end time, you can pass a date or a date w/ time as a string:
+
+```bash
+code42 security-data print -b '2020-02-02 12:51:00'
+```
+
+```bash
+code42 security-data print -b 2020-02-02
+```
+
+or a shorthand string specifying either days, hours, or minutes back from the current time:
+
+```bash
+code42 security-data print -b 30d
+```
 
 ```bash
-code42 security-data print -b 2020-02-02 12:51
+code42 security-data print -b 10d -e 12h
 ```
 
 Begin date will be ignored if provided on subsequent queries using `-i`.

src/code42cli/cmds/securitydata/date_helper.py

@@ -1,10 +1,20 @@
+import re
 from datetime import datetime, timedelta
 
 from c42eventextractor.common import convert_datetime_to_timestamp
 from py42.sdk.queries.fileevents.filters.event_filter import EventTimestamp
 
 _MAX_LOOK_BACK_DAYS = 90
-_FORMAT_VALUE_ERROR_MESSAGE = u"input must be a date in YYYY-MM-DD or YYYY-MM-DD HH:MM:SS format."
+_FORMAT_VALUE_ERROR_MESSAGE = u"input must be a date in YYYY-MM-DD or YYYY-MM-DD HH:MM:SS format, or a short value in days, hours, or minutes (e.g. 30d, 24h, 15m)"
+
+
+class DateArgumentException(Exception):
+    def __init__(self, message=_FORMAT_VALUE_ERROR_MESSAGE):
+        super(DateArgumentException, self).__init__(message)
+
+
+TIMESTAMP_REGEX = re.compile(u"(\d{4}-\d{2}-\d{2})\s*(.*)?")
+MAGIC_TIME_REGEX = re.compile(u"(\d+)([dhm])$")
 
 
 def create_event_timestamp_filter(begin_date=None, end_date=None):
@@ -14,34 +24,20 @@ def create_event_timestamp_filter(begin_date=None, end_date=None):
             begin_date: The begin date for the range.
             end_date: The end date for the range.
     """
-
     if begin_date and end_date:
         min_timestamp = _parse_min_timestamp(begin_date)
         max_timestamp = _parse_max_timestamp(end_date)
         return _create_in_range_filter(min_timestamp, max_timestamp)
+
     elif begin_date and not end_date:
         min_timestamp = _parse_min_timestamp(begin_date)
         return _create_on_or_after_filter(min_timestamp)
+
     elif end_date and not begin_date:
         max_timestamp = _parse_max_timestamp(end_date)
         return _create_on_or_before_filter(max_timestamp)
 
 
-def _parse_max_timestamp(end_date):
-    if len(end_date) == 1:
-        end_date = _get_end_date_with_eod_time_if_needed(end_date)
-        max_time = _parse_timestamp(end_date)
-        max_time = _add_milliseconds(max_time)
-    else:
-        max_time = _parse_timestamp(end_date)
-
-    return convert_datetime_to_timestamp(max_time)
-
-
-def _add_milliseconds(max_time):
-    return max_time + timedelta(milliseconds=999)
-
-
 def _create_in_range_filter(min_timestamp, max_timestamp):
     _verify_timestamp_order(min_timestamp, max_timestamp)
     return EventTimestamp.in_range(min_timestamp, max_timestamp)
@@ -55,45 +51,79 @@ def _create_on_or_before_filter(max_timestamp):
     return EventTimestamp.on_or_before(max_timestamp)
 
 
-def _get_end_date_with_eod_time_if_needed(end_date):
-    return end_date[0], "23:59:59"
+def _parse_timestamp(date_str, rounding_func):
+    timestamp_match = TIMESTAMP_REGEX.match(date_str)
+    magic_match = MAGIC_TIME_REGEX.match(date_str)
+
+    if timestamp_match:
+        date, time = timestamp_match.groups()
+        dt = _get_dt_from_date_time_pair(date, time)
+        if not time:
+            dt = rounding_func(dt)
+
+    elif magic_match:
+        num, period = magic_match.groups()
+        dt = _get_dt_from_magic_time_pair(num, period)
+        if period == u"d":
+            dt = rounding_func(dt)
+
+    else:
+        raise DateArgumentException()
+    return dt
 
 
 def _parse_min_timestamp(begin_date_str):
-    min_time = _parse_timestamp(begin_date_str)
-    min_timestamp = convert_datetime_to_timestamp(min_time)
-    boundary_date = datetime.utcnow() - timedelta(days=_MAX_LOOK_BACK_DAYS)
-    boundary = convert_datetime_to_timestamp(boundary_date)
-    if min_timestamp and min_timestamp < boundary:
-        raise ValueError(u"'Begin date' must be within 90 days.")
-    return min_timestamp
+    dt = _parse_timestamp(begin_date_str, _round_datetime_to_day_start)
+
+    boundary_date = _round_datetime_to_day_start(
+        datetime.utcnow() - timedelta(days=_MAX_LOOK_BACK_DAYS)
+    )
+    if dt < boundary_date:
+        raise DateArgumentException(u"'Begin date' must be within 90 days.")
+
+    return convert_datetime_to_timestamp(dt)
+
+
+def _parse_max_timestamp(end_date_str):
+    dt = _parse_timestamp(end_date_str, _round_datetime_to_day_end)
+    return convert_datetime_to_timestamp(dt)
+
+
+def _get_dt_from_date_time_pair(date, time):
+    date_format = u"%Y-%m-%d %H:%M:%S"
+    time = time or u"00:00:00"
+    date_string = u"{} {}".format(date, time)
+    try:
+        dt = datetime.strptime(date_string, date_format)
+    except ValueError:
+        raise DateArgumentException()
+    else:
+        return dt
+
+
+def _get_dt_from_magic_time_pair(num, period):
+    num = int(num)
+    if period == u"d":
+        dt = datetime.utcnow() - timedelta(days=num)
+    elif period == u"h":
+        dt = datetime.utcnow() - timedelta(hours=num)
+    elif period == u"m":
+        dt = datetime.utcnow() - timedelta(minutes=num)
+    else:
+        raise DateArgumentException(u"Couldn't parse magic time string: {}{}".format(num, period))
+    return dt
 
 
 def _verify_timestamp_order(min_timestamp, max_timestamp):
     if min_timestamp is None or max_timestamp is None:
         return
     if min_timestamp >= max_timestamp:
-        raise ValueError(u"Begin date cannot be after end date")
+        raise DateArgumentException(u"Begin date cannot be after end date")
 
 
-def _parse_timestamp(date_and_time):
-    try:
-        date_str = _join_date_and_time(date_and_time)
-        date_format = u"%Y-%m-%d" if len(date_and_time) == 1 else u"%Y-%m-%d %H:%M:%S"
-        time = datetime.strptime(date_str, date_format)
-        return time
-    except ValueError:
-        raise ValueError(_FORMAT_VALUE_ERROR_MESSAGE)
+def _round_datetime_to_day_start(dt):
+    return dt.replace(hour=0, minute=0, second=0, microsecond=0)
 
 
-def _join_date_and_time(date_and_time):
-    if not date_and_time:
-        return None
-    date_str = date_and_time[0]
-    if len(date_and_time) == 1:
-        return date_str
-    if len(date_and_time) == 2:
-        date_str = "{0} {1}".format(date_str, date_and_time[1])
-    else:
-        raise ValueError(_FORMAT_VALUE_ERROR_MESSAGE)
-    return date_str
+def _round_datetime_to_day_end(dt):
+    return dt.replace(hour=23, minute=59, second=59, microsecond=999000)

src/code42cli/cmds/securitydata/extraction.py

@@ -117,10 +117,10 @@ def _create_filters(args):
 
 def _get_event_timestamp_filter(begin_date, end_date):
     try:
-        begin_date = begin_date.strip().split() if begin_date else None
-        end_date = end_date.strip().split() if end_date else None
+        begin_date = begin_date.strip() if begin_date else None
+        end_date = end_date.strip() if end_date else None
         return date_helper.create_event_timestamp_filter(begin_date, end_date)
-    except ValueError as ex:
+    except date_helper.DateArgumentException as ex:
         print_error(str(ex))
         exit(1)
 

src/code42cli/cmds/securitydata/main.py

@@ -106,13 +106,17 @@ def _load_search_args(arg_collection):
             u"-b",
             u"--{}".format(enums.SearchArguments.BEGIN_DATE),
             help=u"The beginning of the date range in which to look for events, "
-            u"in YYYY-MM-DD (UTC) or YYYY-MM-DD HH:MM:SS (UTC+24-hr time) format.",
+            u"can be a date/time in YYYY-MM-DD (UTC) or YYYY-MM-DD HH:MM:SS (UTC+24-hr time) format "
+            u"or a short value representing days (30d), hours (24h) or minutes (15m) from current "
+            u"time.",
         ),
         enums.SearchArguments.END_DATE: ArgConfig(
             u"-e",
             u"--{}".format(enums.SearchArguments.END_DATE),
             help=u"The end of the date range in which to look for events, "
-            u"in YYYY-MM-DD (UTC) or YYYY-MM-DD HH:MM:SS (UTC+24-hr time) format.",
+            u"can be a date/time in YYYY-MM-DD (UTC) or YYYY-MM-DD HH:MM:SS (UTC+24-hr time) format "
+            u"or a short value representing days (30d), hours (24h) or minutes (15m) from current "
+            u"time.",
         ),
         enums.SearchArguments.EXPOSURE_TYPES: ArgConfig(
             u"-t",

tests/cmds/securitydata/conftest.py

@@ -34,10 +34,10 @@ def get_test_date_str(days_ago):
 begin_date_str_with_time = "{0} 3:12:33".format(begin_date_str)
 end_date_str = get_test_date_str(days_ago=10)
 end_date_str_with_time = "{0} 11:22:43".format(end_date_str)
-begin_date_list = [get_test_date_str(days_ago=89)]
-begin_date_list_with_time = [get_test_date_str(days_ago=89), "3:12:33"]
-end_date_list = [get_test_date_str(days_ago=10)]
-end_date_list_with_time = [get_test_date_str(days_ago=10), "11:22:43"]
+begin_date_str = get_test_date_str(days_ago=89)
+begin_date_with_time = [get_test_date_str(days_ago=89), "3:12:33"]
+end_date_str = get_test_date_str(days_ago=10)
+end_date_with_time = [get_test_date_str(days_ago=10), "11:22:43"]
 
 
 @pytest.fixture(autouse=True)

tests/cmds/securitydata/test_date_helper.py

@@ -1,11 +1,14 @@
 import pytest
-from code42cli.cmds.securitydata.date_helper import create_event_timestamp_filter
+from code42cli.cmds.securitydata.date_helper import (
+    create_event_timestamp_filter,
+    DateArgumentException,
+)
 
 from .conftest import (
-    begin_date_list,
-    begin_date_list_with_time,
-    end_date_list,
-    end_date_list_with_time,
+    begin_date_str,
+    begin_date_with_time,
+    end_date_str,
+    end_date_with_time,
     get_filter_value_from_json,
     get_test_date_str,
 )
@@ -22,57 +25,87 @@ def test_create_event_timestamp_filter_when_given_nones_returns_none():
 
 
 def test_create_event_timestamp_filter_builds_expected_query():
-    ts_range = create_event_timestamp_filter(begin_date_list)
+    ts_range = create_event_timestamp_filter(begin_date_str)
     actual = get_filter_value_from_json(ts_range, filter_index=0)
-    expected = "{0}T00:00:00.000Z".format(begin_date_list[0])
+    expected = "{0}T00:00:00.000Z".format(begin_date_str)
     assert actual == expected
 
 
 def test_create_event_timestamp_filter_when_given_begin_with_time_builds_expected_query():
-    ts_range = create_event_timestamp_filter(begin_date_list_with_time)
+    time_str = u"{} {}".format(*begin_date_with_time)
+    ts_range = create_event_timestamp_filter(time_str)
     actual = get_filter_value_from_json(ts_range, filter_index=0)
-    expected = "{0}T0{1}.000Z".format(begin_date_list_with_time[0], begin_date_list_with_time[1])
+    expected = "{0}T0{1}.000Z".format(*begin_date_with_time)
     assert actual == expected
 
 
 def test_create_event_timestamp_filter_when_given_end_builds_expected_query():
-    ts_range = create_event_timestamp_filter(begin_date_list, end_date_list)
+    ts_range = create_event_timestamp_filter(begin_date_str, end_date_str)
     actual = get_filter_value_from_json(ts_range, filter_index=1)
-    expected = "{0}T23:59:59.999Z".format(end_date_list[0])
+    expected = "{0}T23:59:59.999Z".format(end_date_str)
     assert actual == expected
 
 
 def test_create_event_timestamp_filter_when_given_end_with_time_builds_expected_query():
-    ts_range = create_event_timestamp_filter(begin_date_list, end_date_list_with_time)
+    end_date_str = "{} {}".format(*end_date_with_time)
+    ts_range = create_event_timestamp_filter(begin_date_str, end_date_str)
     actual = get_filter_value_from_json(ts_range, filter_index=1)
-    expected = "{0}T{1}.000Z".format(end_date_list_with_time[0], end_date_list_with_time[1])
+    expected = "{0}T{1}.000Z".format(*end_date_with_time)
     assert actual == expected
 
 
 def test_create_event_timestamp_filter_when_given_both_begin_and_end_builds_expected_query():
-    ts_range = create_event_timestamp_filter(begin_date_list, end_date_list_with_time)
+    end_date = "{} {}".format(*end_date_with_time)
+    ts_range = create_event_timestamp_filter(begin_date_str, end_date)
     actual_begin = get_filter_value_from_json(ts_range, filter_index=0)
     actual_end = get_filter_value_from_json(ts_range, filter_index=1)
-    expected_begin = "{0}T00:00:00.000Z".format(begin_date_list[0])
-    expected_end = "{0}T{1}.000Z".format(end_date_list_with_time[0], end_date_list_with_time[1])
+    expected_begin = "{0}T00:00:00.000Z".format(begin_date_str)
+    expected_end = "{0}T{1}.000Z".format(*end_date_with_time)
     assert actual_begin == expected_begin
     assert actual_end == expected_end
 
 
 def test_create_event_timestamp_filter_when_begin_more_than_ninety_days_back_causes_value_error():
-    begin_date_tuple = (get_test_date_str(days_ago=91),)
-    with pytest.raises(ValueError):
-        create_event_timestamp_filter(begin_date_tuple)
+    begin_date_str = get_test_date_str(days_ago=91)
+    with pytest.raises(DateArgumentException):
+        create_event_timestamp_filter(begin_date_str)
 
 
 def test_create_event_timestamp_filter_when_end_is_before_begin_causes_value_error():
-    begin_date_tuple = (get_test_date_str(days_ago=5),)
-    end_date_str = (get_test_date_str(days_ago=7),)
-    with pytest.raises(ValueError):
-        create_event_timestamp_filter(begin_date_tuple, end_date_str)
+    begin_date = get_test_date_str(days_ago=5)
+    end_date = get_test_date_str(days_ago=7)
+    with pytest.raises(DateArgumentException):
+        create_event_timestamp_filter(begin_date, end_date)
+
+
+def test_create_event_timestamp_filter_when_args_are_magic_days_builds_expected_query():
+    begin_magic_str = "10d"
+    end_magic_str = "6d"
+    ts_range = create_event_timestamp_filter(begin_magic_str, end_magic_str)
+    actual_begin = get_filter_value_from_json(ts_range, filter_index=0)
+    expected_begin = "{}T00:00:00.000Z".format(get_test_date_str(days_ago=10))
+    actual_end = get_filter_value_from_json(ts_range, filter_index=1)
+    expected_end = "{}T23:59:59.999Z".format(get_test_date_str(days_ago=6))
+    assert actual_begin == expected_begin
+    assert actual_end == expected_end
 
 
-def test_create_event_timestamp_filter_when_given_three_date_args_raises_value_error():
-    begin_date_tuple = (get_test_date_str(days_ago=5), "12:00:00", "end_date=12:00:00")
-    with pytest.raises(ValueError):
-        create_event_timestamp_filter(begin_date_tuple)
+def test_create_event_timestamp_filter_when_given_improperly_formatted_arg_raises_value_error():
+    missing_seconds = "{} {}".format(get_test_date_str(days_ago=5), "12:00")
+    month_first_date = "01-01-2020"
+    time_typo = "{} {}".format(get_test_date_str(days_ago=5), "b20:30:00")
+    bad_magic = "2months"
+    bad_magic_2 = "100s"
+    bad_magic_3 = "10 d"
+    with pytest.raises(DateArgumentException):
+        create_event_timestamp_filter(missing_seconds)
+    with pytest.raises(DateArgumentException):
+        create_event_timestamp_filter(month_first_date)
+    with pytest.raises(DateArgumentException):
+        create_event_timestamp_filter(time_typo)
+    with pytest.raises(DateArgumentException):
+        create_event_timestamp_filter(bad_magic)
+    with pytest.raises(DateArgumentException):
+        create_event_timestamp_filter(bad_magic_2)
+    with pytest.raises(DateArgumentException):
+        create_event_timestamp_filter(bad_magic_3)