From a8cdc2613208d13f8f5abeca5a104bb90c89e6bb Mon Sep 17 00:00:00 2001 From: Zach Williams Date: Sat, 12 Jan 2013 23:12:20 -0800 Subject: [PATCH 1/8] Update README to use Markdown format --- README.asciidoc => README.md | 57 +++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 20 deletions(-) rename README.asciidoc => README.md (85%) diff --git a/README.asciidoc b/README.md similarity index 85% rename from README.asciidoc rename to README.md index 75d48fd..dfb2290 100644 --- a/README.asciidoc +++ b/README.md @@ -1,21 +1,26 @@ -= libfitbit = +libfitbit +========= by Kyle Machulis -Nonpolynomial Labs - http://www.nonpolynomial.com +Nonpolynomial Labs - [http://www.nonpolynomial.com](http://www.nonpolynomial.com) libfitbit is part of the OpenYou project for Open Source Health -Hardware access - http://openyou.org +Hardware access - [http://openyou.org](http://openyou.org) If you find libfitbit useful, please donate to the project at -http://pledgie.com/campaigns/14375 +[http://pledgie.com/campaigns/14375](http://pledgie.com/campaigns/14375) -== Credits and Thanks == + +Credits and Thanks +------------------ Thanks to Matt Cutts for hooking me up with the hardware - http://www.twitter.com/mattcutts -== Description == + +Description +----------- libfitbit is an implementation of the data retrieval protocol for the fitbit health tracking device. It also implements a synchronization @@ -38,32 +43,40 @@ be used for implementing the protocol in new languages without having to read code (not that my ability to convey the protocol in english is all that clear). -== Package Information == -Source repo @ http://www.github.com/qdot/libfitbit +Package Information +------------------- + +Source repo @ [http://www.github.com/qdot/libfitbit](http://www.github.com/qdot/libfitbit) + -== Platform Support == +Platform Support +---------------- * Linux - Tested on Ubuntu 10.10 * OS X - Untested, should work? * Windows - Won't work at the moment. May be able to create serial interface to talk to CP2012 chip? Haven't done research yet. -== Library Requirements == + +Library Requirements +-------------------- * Python - http://www.python.org * libusb-1.0 - http://www.libusb.org * pyusb 1.0+ - http://sourceforge.net/projects/pyusb/files/ -== Platform Cavaets == -=== Linux === +Platform Cavaets +---------------- + +### Linux You'll need to either run as root or set up a udev rule to switch out permissions on the base VID/PID. We'll hopefully have a udev rule checked in shortly. -=== OS X === +### OS X FitBit original driver is claiming the device resulting in premission errors when libusb wants to claims it. @@ -71,29 +84,33 @@ when libusb wants to claims it. A solution that works everytime is simply to disable the driver by renaming it: ----- +```bash cd /System/Library/Extensions sudo mv SiLabsUSBDriver.kext SiLabsUSBDriver.kext.disabled ----- +``` And reboot. To re-enable it, just rename it again, and reboot again. -=== Windows === +### Windows Don't even know if it works there yet. :D -== Future Plans == + +Future Plans +------------ * Breaking ANT access library out into its own repo * Windows support * Finish figuring out data types * Implement library in C -== License == ---------------------- +License +------- + +``` Copyright (c) 2011, Kyle Machulis/Nonpolynomial Labs All rights reserved. @@ -126,4 +143,4 @@ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------- +``` From f5044ed1352238dcc9701dc61f4e5b12d17eb5e6 Mon Sep 17 00:00:00 2001 From: bugramovic Date: Sun, 13 Jan 2013 20:15:13 +0100 Subject: [PATCH 2/8] added a csv export for the fitbit-data (minute-data: steps, meters, floor plus daily statistics) --- python/csv_writer.py | 246 ++++++++++++++++++++++++++++++++++++++++ python/fitbit_client.py | 20 +++- 2 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 python/csv_writer.py diff --git a/python/csv_writer.py b/python/csv_writer.py new file mode 100644 index 0000000..0f2b482 --- /dev/null +++ b/python/csv_writer.py @@ -0,0 +1,246 @@ +################################################################# +# csv writer +# export the fitbit data into csv files under ~/.fitbit/id/csv +# so they can be read by other software packages (e.g. gnuplot) +# +# +# Distributed as part of the libfitbit project +# +# Repo: https://github.com/benallard/libfitbit +# +# Licensed under the BSD License, as follows +# +# +# Redistribution and use in source and binary forms, +# with or without modification, are permitted provided +# that the following conditions are met: +# +# * Redistributions of source code must retain the +# above copyright notice, this list of conditions +# and the following disclaimer. +# * Redistributions in binary form must reproduce the +# above copyright notice, this list of conditions and +# the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# * Neither the name of the Nonpolynomial Labs nor the names +# of its contributors may be used to endorse or promote +# products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +################################################################# + +import os, csv, yaml, datetime, itertools + +ENABLE_LOGGING = True + +def _log(msg): + if ENABLE_LOGGING: + print "csv_writer: " + str(msg) + +def _read_yaml(yaml_file_path): + f = open(yaml_file_path) + raw_data = f.read() + f.close() + result = yaml.load(raw_data) + return result + +def _get_flat_req_resp_list(data): + if (len(data)>1): + result = [] + for outer_list in data: + for inner_list in outer_list: + for req_resp in inner_list: + if "request" in req_resp: + result.append(inner_list) + + return result + else: + raise Exception("data does not have the expected format") + +def _filter_by_opcodes(req_resp_list, filter_function): + """ + filter_function receives the opcodes as first and only parameter and returns a boolean + returns a list with all requests where filter_function returns true + """ + result = [] + for reqresp in req_resp_list: + req = reqresp["request"] + opcode = req['opcode'] + include = filter_function(opcode) + if (include): + result.append(reqresp) + + return result + +def _p_0(data): + i = 0 + tstamp = 0 + result = [] + while i < len(data): + if (data[i] & 0x80) == 0x80: + d = data[i:i+3] + d0 = d[0] - 0x80 # ??? + d1 = (d[1] - 10) / 10. # score + d2 = d[2] # steps + row = {'timestamp':tstamp, 'datetime':datetime.datetime.fromtimestamp(tstamp), '?':d0, 'score':d1, 'steps':d2} + _log(row) + result.append(row) + i += 3 + tstamp += 60 + continue + d = data[i:i+4] + tstamp = d[3] | d[2] << 8 | d[1] << 16 | d[0] << 24 + if i != 0: _log('---xx--xx--xx--xx') + i += 4 + return result + +def _p_1(data): + assert len(data) % 16 == 0 + result = [] + for i in xrange(0, len(data), 16): + d = data[i:i+16] + tstamp = d[0] | d[1] << 8 | d[2] << 16 | d[3] << 24 + #if date in seen: continue + #seen.add(date) + date = datetime.datetime.fromtimestamp(tstamp) + if date.minute == 0 and date.hour == 0 and date.second == 0: + #print a2s(d[4:]) + daily_steps = d[7] << 8 | d[6] + daily_dist = (d[11] << 8 | d[10] | d[13] << 24 | d[12] << 16) / 1000000. + daily_floors = (d[15] << 8 | d[14]) / 10 + daily_cals = (d[5] << 8 | d[4]) *.1103 - 7 + row = {'timestamp':tstamp, 'datetime':date, 'steps': daily_steps, 'distance': daily_dist,'floors':daily_floors, 'calories':daily_cals} + _log(row) + result.append(row) + + return result + +def _p_6(data): + result = [] + i = 0 + tstamp = 0 + while i < len(data): + if data[i] == 0x80: + floors = data[i+1] / 10 + row = {'timestamp':tstamp, 'datetime':datetime.datetime.fromtimestamp(tstamp), 'floors':floors} + _log(row) + result.append(row) + i += 2 + tstamp += 60 + continue + + d = data[i:i+4] + tstamp = d[3] | d[2] << 8 | d[1] << 16 | d[0] << 24 + i += 4 + return result + +def convert_for_csv(data): + """ + returns a dict with 'minute_activity'-, 'daily_stats' and 'minute_floors'-data + """ + result = {} + request_response_list = _get_flat_req_resp_list(data) + p0 = _filter_by_opcodes(request_response_list, lambda opcode: (opcode[0] == 0x22 and opcode[1] == 0x00) ) + minute_activity = map( _p_0, ( map(lambda e: e['response'], p0) ) ) + _log( minute_activity ) + + p1 = _filter_by_opcodes(request_response_list, lambda opcode: (opcode[0] == 0x22 and opcode[1] == 0x01)) + daily_stats = map( _p_1, ( map(lambda e: e['response'], p1) ) ) + _log( daily_stats ) + + p6 = _filter_by_opcodes(request_response_list, lambda opcode: (opcode[0] == 0x22 and opcode[1] == 0x06)) + minute_floors = map( _p_6, ( map(lambda e: e['response'], p6) ) ) + _log( minute_activity ) + + result['minute_activity'] = list(itertools.chain.from_iterable(minute_activity)) + result['daily_stats'] = list(itertools.chain.from_iterable(daily_stats)) + result['minute_floors'] = list(itertools.chain.from_iterable(minute_floors)) + + return result + +def _write_csv_file(directory, filename, header, rows): + """ + uses a csv.DictWriter to write the rows (list of dicts) into a csv-file (in the order given in header) + """ + csv_file_path = os.path.join(directory, filename) + is_new_csv = (not os.path.exists(csv_file_path)) + if is_new_csv: + f = open(csv_file_path, 'wb') + else: + f = open(csv_file_path, 'ab') + + writer = csv.DictWriter(f, header, delimiter=';') + if is_new_csv: + writer.writeheader() + + writer.writerows(rows) + f.close() + +def write_csv(converted_data, tracker_id, directory='~/.fitbit'): + directory = os.path.expanduser(directory) + directory = os.path.join(directory, tracker_id) + directory = os.path.join(directory, 'csv') + if not os.path.isdir(directory): + os.makedirs(directory) + + _write_csv_file(directory, "minute_activity.csv", + ['timestamp', 'datetime', '?', 'score', 'steps'], converted_data['minute_activity']) + + _write_csv_file(directory, "minute_floors.csv", + ['timestamp', 'datetime', 'floors'], converted_data['minute_floors']) + _write_csv_file(directory, "daily_stats.csv", + ['timestamp', 'datetime', 'steps', 'distance', 'floors', 'calories'], converted_data['daily_stats'] ) + +def convert_dump_to_csv(yaml_file_path, tracker_id, directory='~/.fitbit'): + data = _read_yaml(yaml_file_path) + converted = convert_for_csv(data) + write_csv(converted, tracker_id) + + +def main(): + """ + Finds the most recent connection-dump and tries to convert it + write CSV + Used for testing + """ + directory = os.path.expanduser('~/.fitbit') + dirlist = os.listdir(directory) + if (len(dirlist) == 0): + _log("No Tracker-Directory found. Aborting.") + + tracker_id = dirlist[0] #use first tracker + + directory = os.path.join(directory, tracker_id) #use first tracker + + + most_recent = None + most_recent_time = 0.0 + for fname in os.listdir(directory): + if 'connection-' in fname: + full_path = os.path.join(directory, fname) + time = os.path.getctime(full_path) + if (time > most_recent_time): + most_recent = full_path + most_recent_time = time + + if most_recent: + convert_dump_to_csv(most_recent, tracker_id) + else: + _log("Found no connection dump. Aborting.") + + _log("done") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/python/fitbit_client.py b/python/fitbit_client.py index 8df4adc..27e5d01 100755 --- a/python/fitbit_client.py +++ b/python/fitbit_client.py @@ -55,6 +55,7 @@ import argparse import xml.etree.ElementTree as et from fitbit import FitBit +import csv_writer from antprotocol.connection import getConn from antprotocol.protocol import ANT, ANTException, FitBitBeaconTimeout @@ -182,21 +183,34 @@ def form_base_info(self, remote_info=None): for f in ['deviceInfo.serialNumber','userPublicId']: if f in self.info_dict: self.log_info[f] = self.info_dict[f] + def dump_connection(self, directory='~/.fitbit'): directory = os.path.expanduser(directory) + directory = os.path.join(directory, self.log_info['userPublicId']) + output_file = os.path.join(directory,'connection-%d.txt' % int(self.time)) data = yaml.dump(self.data) - if 'userPublicId' in self.log_info: - directory = os.path.join(directory, self.log_info['userPublicId']) + if 'userPublicId' in self.log_info: if not os.path.isdir(directory): os.makedirs(directory) - f = open(os.path.join(directory,'connection-%d.txt' % int(self.time)), 'w') + f = open(output_file, 'w') f.write(data) f.close() print data + return output_file + + def write_csv(self): + import traceback + try: + csv_writer.write_csv( csv_writer.convert_for_csv(self.data), self.info_dict['userPublicId'] ) + except Exception: + print "Could not write csv files." + traceback.print_exc(file=sys.stdout) def close(self): self.dump_connection() + self.write_csv() + print 'Closing USB device' try: self.fitbit.base.connection.close() From d830b813a09d937bc4ad3aa1e01f3342d1947b91 Mon Sep 17 00:00:00 2001 From: bugramovic Date: Mon, 14 Jan 2013 22:16:19 +0100 Subject: [PATCH 3/8] fixed: accessed the wrong dictionary --- python/fitbit_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/fitbit_client.py b/python/fitbit_client.py index 27e5d01..33f5db6 100755 --- a/python/fitbit_client.py +++ b/python/fitbit_client.py @@ -187,7 +187,6 @@ def form_base_info(self, remote_info=None): def dump_connection(self, directory='~/.fitbit'): directory = os.path.expanduser(directory) - directory = os.path.join(directory, self.log_info['userPublicId']) output_file = os.path.join(directory,'connection-%d.txt' % int(self.time)) data = yaml.dump(self.data) if 'userPublicId' in self.log_info: @@ -202,7 +201,8 @@ def dump_connection(self, directory='~/.fitbit'): def write_csv(self): import traceback try: - csv_writer.write_csv( csv_writer.convert_for_csv(self.data), self.info_dict['userPublicId'] ) + if 'userPublicId' in self.log_info: + csv_writer.write_csv( csv_writer.convert_for_csv(self.data), self.log_info['userPublicId'] ) except Exception: print "Could not write csv files." traceback.print_exc(file=sys.stdout) From 3a01531b9c0615e312249629e1a1b4c0aae2dd55 Mon Sep 17 00:00:00 2001 From: bugramovic Date: Mon, 14 Jan 2013 22:49:22 +0100 Subject: [PATCH 4/8] test-mode of the csv-writer now also works with files in the ~/.fitbit directory --- python/csv_writer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/csv_writer.py b/python/csv_writer.py index 0f2b482..54b38e6 100644 --- a/python/csv_writer.py +++ b/python/csv_writer.py @@ -215,11 +215,13 @@ def main(): Finds the most recent connection-dump and tries to convert it + write CSV Used for testing """ - directory = os.path.expanduser('~/.fitbit') - dirlist = os.listdir(directory) + directory = os.path.expanduser('~/.fitbit') + dirlist = filter(lambda name: os.path.isdir( os.path.join(directory, name) ), os.listdir(directory)) + if (len(dirlist) == 0): _log("No Tracker-Directory found. Aborting.") + tracker_id = dirlist[0] #use first tracker directory = os.path.join(directory, tracker_id) #use first tracker From 975adeab155100f876863a0e6235506967cde621 Mon Sep 17 00:00:00 2001 From: bugramovic Date: Mon, 14 Jan 2013 23:07:57 +0100 Subject: [PATCH 5/8] added a config file reader. default to no output (no connection-dump, no csv) --- python/client_config.py | 22 ++++++++++++++++++++++ python/fitbit_client.py | 11 +++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 python/client_config.py diff --git a/python/client_config.py b/python/client_config.py new file mode 100644 index 0000000..77401d5 --- /dev/null +++ b/python/client_config.py @@ -0,0 +1,22 @@ +import ConfigParser, os + +class ClientConfig(object): + """ + Example-Configuration + + [output] + dump_connection = True + write_csv = False + """ + + def __init__(self): + self.parser = ConfigParser.SafeConfigParser({'dump_connection':False, 'write_csv':False}) + config_path = os.path.expanduser('~/.fitbit/config') + if os.path.exists(config_path): + self.parser.read(config_path) + + def dump_connection(self): + return self.parser.getboolean('output', 'dump_connection') + + def write_csv(self): + return self.parser.getboolean('output', 'write_csv') diff --git a/python/fitbit_client.py b/python/fitbit_client.py index 33f5db6..14ac2fa 100755 --- a/python/fitbit_client.py +++ b/python/fitbit_client.py @@ -55,7 +55,7 @@ import argparse import xml.etree.ElementTree as et from fitbit import FitBit -import csv_writer +import csv_writer, client_config from antprotocol.connection import getConn from antprotocol.protocol import ANT, ANTException, FitBitBeaconTimeout @@ -208,8 +208,11 @@ def write_csv(self): traceback.print_exc(file=sys.stdout) def close(self): - self.dump_connection() - self.write_csv() + cfg = client_config.ClientConfig() + if cfg.dump_connection(): + self.dump_connection() + if cfg.write_csv(): + self.write_csv() print 'Closing USB device' try: @@ -270,7 +273,7 @@ def try_sync(self): print '-'*60 traceback.print_exc(file=sys.stdout) print '-'*60 - self.write_log('ERROR: ' + str(e)) + elf.write_log('ERROR: ' + str(e)) self.errors += 1 except usb.USBError, e: # Raise this error up the stack, since USB errors are fairly From 4edae3217da29b0507c17882c6b7d0c2ddef8eb5 Mon Sep 17 00:00:00 2001 From: bugramovic Date: Mon, 14 Jan 2013 23:15:11 +0100 Subject: [PATCH 6/8] accidently removed a character --- python/fitbit_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/fitbit_client.py b/python/fitbit_client.py index 14ac2fa..57e1fb0 100755 --- a/python/fitbit_client.py +++ b/python/fitbit_client.py @@ -273,7 +273,7 @@ def try_sync(self): print '-'*60 traceback.print_exc(file=sys.stdout) print '-'*60 - elf.write_log('ERROR: ' + str(e)) + self.write_log('ERROR: ' + str(e)) self.errors += 1 except usb.USBError, e: # Raise this error up the stack, since USB errors are fairly From 90f776c2a9cb93c2faa095020cbd9b0f155924ff Mon Sep 17 00:00:00 2001 From: bugramovic Date: Tue, 15 Jan 2013 00:56:21 +0100 Subject: [PATCH 7/8] fixed merge conflicts :-/ --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 43154d3..7cda677 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ Hardware access - [http://openyou.org](http://openyou.org) If you find libfitbit useful, please donate to the project at [http://pledgie.com/campaigns/14375](http://pledgie.com/campaigns/14375) -<<<<<<< HEAD:README.asciidoc Join the libfitbit group on fitbit.com: http://www.fitbit.com/group/227FRX == Credits and Thanks == @@ -19,8 +18,6 @@ Join the libfitbit group on fitbit.com: http://www.fitbit.com/group/227FRX Credits and Thanks ------------------ ->>>>>>> a8cdc2613208d13f8f5abeca5a104bb90c89e6bb:README.md - Thanks to Matt Cutts for hooking me up with the hardware - http://www.twitter.com/mattcutts From cca55a1c77712462af78ee61d504639b14d17457 Mon Sep 17 00:00:00 2001 From: bugramovic Date: Tue, 15 Jan 2013 01:09:27 +0100 Subject: [PATCH 8/8] fixed readme headline --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 7cda677..2bf0f2d 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,6 @@ If you find libfitbit useful, please donate to the project at Join the libfitbit group on fitbit.com: http://www.fitbit.com/group/227FRX -== Credits and Thanks == -======= - Credits and Thanks ------------------ Thanks to Matt Cutts for hooking me up with the hardware -