From fc60413403863a5ebad4f3c54a45a4d4b1cf6eff Mon Sep 17 00:00:00 2001 From: muschellij2 Date: Thu, 27 Aug 2020 10:04:08 -0400 Subject: [PATCH 1/5] just harmonizing - no change --- read_raw_gt3x.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/read_raw_gt3x.py b/read_raw_gt3x.py index cc0f409..955895b 100644 --- a/read_raw_gt3x.py +++ b/read_raw_gt3x.py @@ -72,7 +72,7 @@ def process_gt3x_file(idx, total, file, save_folder, delete_source_file, delete_ save_folder = os.path.join(save_folder, subfolder) # unzip .gt3x file and get the file location of the binary log.bin (which contains the raw data) and the info.txt which contains the meta-data - # log_bin, info_txt = unzip_gt3x_file(f = file, save_location = save_folder, delete_source_file = delete_source_file) + # log_bin, info_txt, _ = unzip_gt3x_file(f = file, save_location = save_folder, delete_source_file = delete_source_file) # # get meta data from info.txt file # meta_data = extract_info(info_txt) From 5d75ea3080a135881f36ecd6d43842fd1fcb31ed Mon Sep 17 00:00:00 2001 From: muschellij2 Date: Thu, 27 Aug 2020 10:15:47 -0400 Subject: [PATCH 2/5] updated with new gt3x --- .gitignore | 3 ++- read_data.py | 26 ++++++++++++++++++++++++++ read_raw_gt3x.py | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 read_data.py diff --git a/.gitignore b/.gitignore index 74f11e0..9b7fd4f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .DS_Store *.pyc __pycache__ -logs/ \ No newline at end of file +logs/ +data/ \ No newline at end of file diff --git a/read_data.py b/read_data.py new file mode 100644 index 0000000..43964ba --- /dev/null +++ b/read_data.py @@ -0,0 +1,26 @@ +import os +import numpy as np +import tempfile as tmp + +# import functions +from functions.helper_functions import set_start, set_end, read_directory, create_directory +from gt3x import unzip_gt3x_file, extract_info, extract_log, create_time_array, rescale_log_data + + +file = "data/AI12_NEO1F09120034_2017-09-25.gt3x" +save_folder = tmp.TemporaryDirectory() +save_folder = save_folder.name +delete_source_file = False +# unzip .gt3x file and get the file location of the binary log.bin (which contains the raw data) and the info.txt which contains the meta-data +log_bin, info_txt, _, _ = unzip_gt3x_file(f = file, save_location = save_folder, delete_source_file = delete_source_file) + +# get meta data from info.txt file +meta_data = extract_info(info_txt) + +# read raw data from binary data +log_data, time_data = extract_log(log_bin = log_bin, acceleration_scale = float(meta_data['Acceleration_Scale']), sample_rate = int(meta_data['Sample_Rate']), use_scaling = False) + +actigraph_acc = rescale_log_data(log_data = log_data, acceleration_scale = meta_data['Acceleration_Scale']) + +# convert time data to correct time series array with correct miliseconds values +actigraph_time = create_time_array(time_data) diff --git a/read_raw_gt3x.py b/read_raw_gt3x.py index 955895b..f02b083 100644 --- a/read_raw_gt3x.py +++ b/read_raw_gt3x.py @@ -72,7 +72,7 @@ def process_gt3x_file(idx, total, file, save_folder, delete_source_file, delete_ save_folder = os.path.join(save_folder, subfolder) # unzip .gt3x file and get the file location of the binary log.bin (which contains the raw data) and the info.txt which contains the meta-data - # log_bin, info_txt, _ = unzip_gt3x_file(f = file, save_location = save_folder, delete_source_file = delete_source_file) + # log_bin, info_txt, _, _ = unzip_gt3x_file(f = file, save_location = save_folder, delete_source_file = delete_source_file) # # get meta data from info.txt file # meta_data = extract_info(info_txt) From 4389b2467138212129ac0db030e67908e51f9386 Mon Sep 17 00:00:00 2001 From: muschellij2 Date: Mon, 2 Nov 2020 21:19:25 -0500 Subject: [PATCH 3/5] added allow pickle --- infer_nw_time.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infer_nw_time.py b/infer_nw_time.py index aff6896..2c2e666 100644 --- a/infer_nw_time.py +++ b/infer_nw_time.py @@ -79,6 +79,8 @@ def parse_arguments(): # read file as numpy array data = np.load(file) + data.allow_pickle = True; + # extract raw acceleration data from numpy. actigraph_acc = data['raw_data'] From 95cff96af138c9e073a6dd64ce15f43edadbb178 Mon Sep 17 00:00:00 2001 From: muschellij2 Date: Mon, 2 Nov 2020 21:34:57 -0500 Subject: [PATCH 4/5] now meta_data is saved as a list --- infer_nw_time.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infer_nw_time.py b/infer_nw_time.py index 2c2e666..f1f73b9 100644 --- a/infer_nw_time.py +++ b/infer_nw_time.py @@ -86,6 +86,8 @@ def parse_arguments(): # read meta data from file meta_data = data['meta_data'] + if type(meta_data) is np.ndarray: + meta_data = meta_data.tolist() # convert acceleration values to g values actigraph_acc = rescale_log_data(log_data = actigraph_acc, acceleration_scale = meta_data['Acceleration_Scale']) From 04d8cd5b9886b586c8ae3032f308fba56e14149b Mon Sep 17 00:00:00 2001 From: muschellij2 Date: Sun, 29 Nov 2020 11:45:57 -0500 Subject: [PATCH 5/5] added resampy for sig to work --- functions/signal_processing_functions.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/functions/signal_processing_functions.py b/functions/signal_processing_functions.py index 839ab3f..65bc730 100644 --- a/functions/signal_processing_functions.py +++ b/functions/signal_processing_functions.py @@ -3,10 +3,16 @@ """ import sys import logging -import resampy # to resample frequency + import numpy as np from scipy import signal -from multiprocessing import cpu_count +import resampy +import math +try: + from multiprocessing import cpu_count +except ImportError: + def cpu_count(): + return 1 try: from joblib import Parallel @@ -100,6 +106,7 @@ def resample_acceleration(data, from_hz, to_hz, use_parallel = False, num_jobs = # calculate number of 1 sec samples (note that hz is the frequency per second) num_seconds = len(data) // from_hz + # num_seconds = len(data)/from_hz) # calculate number of new samples required when data is resampled num_samples = num_seconds * to_hz