diff --git a/.travis.yml b/.travis.yml index 1fe79db..e06bb43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ cache: - $HOME/pypmi-data python: - - 3.6 - 3.7 + - 3.8 env: matrix: @@ -20,16 +20,16 @@ env: global: - INSTALL_TYPE=setup - PPMI_PATH=$HOME/pypmi-data - - secure: "L+LQhRYxaT/0VRnbfxYQ3K/qf//g1kH0Rrn1P1blNFzUUrMkBhsRvusZCkinvze54pi2G6zXs2QvFFSKU5fvkrTiSdfbOiXHBJ5oOOAOG9dehyy2zr2rkzBinDdoEOygiuXQdVKM+JTK1gc3wGASp2TjKbEON341c6jYVIX2SZ8nFfcsfyyNh5Cxf0XC017gZuLi3wqfWFlyyCzbwIPTUq88YI+LroaBmE2C5BtZFC4IHlQsZMzo/SNr8uzJwNXWCi3AGo2QVdgwaHGv4JoJpt+JYlgciFKsIyMkEbZrMLK+wIqtyJdNnode7qPVxzQMVSUaLcHL8usKPmB1Ew5ZSpT0Re257HFsu6HX21tPkkxdztQD0yqMENeM98TSbwEragmrXMl7EjDOEoEzlsZzGVUdGy7Vxz1Ub+pTR/hZRTc0Br4eOhM5AUVrdg/Wl013c14A0y8OOt+xGYvVsUmQX2dgvjhAEXK7AsRwJYCNx5Juuz7gdccF9THO1dGfPcKtGPeFPC5FsxYhYr0ZSzMMDm7/oRRmzxYk2f3w11IlUPGa0sTiZu1BliAVq5r/lhw8UJpB0EgM/UxWUb8bd967//AJM3XqHsdohE+1wyicEfjqPtSnTsvQ5Flrqg3aBxvmcro7fIvKrEJu6ufj6j5mAnGDW/pPBv2zWMCDMvfoDx8=" - - secure: "IDW2SWqYhT0SFi5cBvh/5NpM8o1S+qJYq0+TASOM4Bs6Gpx5Rd6XWtofr87LaU3RZmwcOng2QvGV7JBZa50f9Fnf3ucNzhX5EfP6h2dS8F52yTh0yoM/TN7Al2rIndYF8YH11o+UjrqoQtDmILZlzwZBQcf/0pIuwcPpZBX21iFnaecZihePYCYvhGgDh0RHaZHEbgtPSMW4gDVrxgWB5hahPRKRofnyfpwGdmbpBbXEHY+Avmumt5hmIpmtagrkz2WHrwq+xr1LCG7MGsrgeDxLQtSwrrgAplCvOtIQW9DDz+avY+LEGFCqt6yd/lzwyzllTZyyRsuWd6oRYBkFVJPR5cDM65NSSIY87ZgfUzQHsBPV9D7vjvfhh5A215LTAIzgZ3JLqT6XUjFvjQ7kn6JwvF5dsnNjc4EV7bz0ru3FtJnCFUJzCUPfZ71hl4CHRF/H74XiPijnpjmzVbdQbk1fm48mrKhG6AmKAUYVMDVd2fVRafM8AgWs2MKkyTmddxufmW4LADhof/T1t9qQZsvfRzJ9WVafPnHtTDVvl0BQO2NG3oV62mCWZNEzdOPojYFEsBXPwUMzh+ekq3+m7K/Bsly/dhT+LYcPwvfqbE4K0/mHVgajF0vO3WoGHnZhbVrz0i51YycAjSdWeI1R3RdCOFMwaBTI5e+0fdjP6M0=" + - secure: "ZDv6uahvC2HlI5UwpFegljzsUiKqMZY5a8+H0GXhRKgJQXfhjTE+CD0bcCHTq9Z+T1+ZC1mapjBnsS1kxNs3g2WR2z8pVvaWEiqaTxG+YYNRnouxhqhprzzyiMyE1KI0xomHBg56Ph6k4s82v53G3ewuxsujIa9Htu+fmmDFubBhPNRKMud+BGMoTxR0ZNv4q9tBHd4ezYpB8jFyb6RvBpvVTsLrYuHU6KZdVK58EARNgxVSdxYAOSiBGMGD7kvV15eXvaiUHPBLbBK8k0L5CMVeFJHfb+j+sLVgRpmIcKGbCPZMs/cMX0BEQCZ/P+/MScL/7w1+CkLMkkRcZ8be+8yBvCij8U05bXprnwroHupeJaSG18y37o6F19sUevTE0YqfAkIkWeoLipazdzeAe+Mgy34J5onGaE4EdyjPtl9aysQpjc7FyqVRWMIb37wF5F+Apl/w4E3moHy9fv1EajnMZbbznaZbV/LUfonoixFIwMxbcR1p7wEw0gco5NiQJk0Dm+n7W6IHsLiWf8OkZvlQI4Ctlt7q45P+WA5kL3AM27zULEA1Gm2DrV2rlEmDDXrnfxePvdXQo/iDhCpmt1e8lUQMByg/C3BOHpWgpSFGgLFefhlhfxg5MJJL2/0AHwwJvJgtGVbYbSnjdTB2l+WQ/EPFyYYXxwA9idZL49s=" + - secure: "Nq1Zfdw65IhRhJtWnsUBzn1HcWL4QmxWbafiQI25XkJUg4Ew/W1AaPtdCqR9wW3TRhwDooUtq7a85RrKtnX0Cq97154AF6Nx3pNyLfkgUP6/vVqApOcndFHkApsauFu21gDlbCHd9NokV7CRasiNLaR8jKRnj7c9aGARx2ZvOnK2D9x7wuRwNID8LJzlNtI4QAf0c+2v8YV2dFab08Lh8vNCkH4j3r/Yxy9BXfAZvbYhYWD0MYEEW9fb5i01w0Jd8U/1Ez8i7XPgYP6HgH1erwGRPA+T+fbcUk2cR07FUtucwEFPzcKBs4CxkE9NqgoC/X1fHe+fKzRILxNJ0tXnITeiR/JwgVEaZPL+KrlFKfd2bkjFfLq0wGiKW+g9VoHzKK+E0L0vs0R7+i5JirJq1qy4LnL8dZHS4Iyj4Dz4rQJ5xMruaeHjf1ZU4XZWYiHPq1vo6JtlBPr1so1zPKHIV8Mrph3qZx7I/Qp0hwXY6zJXggaxIGoxwbZjbX7fSNR0QfMyhpDQBSXsBNiTXd8KsX9tjJqOLWkBBfdaLlsfUQqCMJxkIJrmetJ8JZI8nDli41P7QXRond6d4rHmVc5oRgF5Pcfv6kY5ifu44EYfL+tELT26eai59jTGDrpqer/k1FvfgkItYFuyw1XqAZFmMijkqaCxJNZ/NSesZeQbioY=" matrix: include: - - python: 3.6 + - python: 3.7 env: - INSTALL_TYPE=sdist - CHECK_TYPE=test - - python: 3.6 + - python: 3.7 env: - INSTALL_TYPE=wheel - CHECK_TYPE=test @@ -42,11 +42,12 @@ before_install: - if [ "${CHECK_TYPE}" == "test" ]; then pip install "pytest>=3.6" pytest-cov coverage coveralls codecov; fi + - python -m pip install --upgrade numpy install: - | if [ "${INSTALL_TYPE}" == "setup" ]; then - python setup.py install; + pip install ./; elif [ "${INSTALL_TYPE}" == "sdist" ]; then python setup.py sdist; pip install dist/*.tar.gz; diff --git a/pypmi/data/studydata.json b/pypmi/data/studydata.json index 736f2e4..17fbac0 100644 --- a/pypmi/data/studydata.json +++ b/pypmi/data/studydata.json @@ -27,10 +27,6 @@ "id": 55, "name": "Blood_Chemistry___Hematology.csv" }, - "Center-subject list": { - "id": 47, - "name": "Center-Subject_List.csv" - }, "Clinical cognitive categorization": { "id": 401, "name": "Cognitive_Categorization.csv" diff --git a/pypmi/fetchers.py b/pypmi/fetchers.py index d24cef7..73f4ee1 100644 --- a/pypmi/fetchers.py +++ b/pypmi/fetchers.py @@ -97,8 +97,8 @@ def _download_data(info: Dict[str, Dict[str, str]], user: str = None, password: str = None, overwrite: bool = False, - verbose: bool = True, - bundle: bool = True) -> List[str]: + verbose: bool = False, + bundle: bool = False) -> List[str]: """ Downloads dataset(s) listed in `info` from `url` @@ -190,12 +190,14 @@ def _download_data(info: Dict[str, Dict[str, str]], # determine whether we're bundling the data (i.e., requesting all files at # once) or peforming separate downloads + headers = {'Referer': 'https://ida.loni.usc.edu/'} if bundle: file_ids = [file_ids] for fid in file_ids: params['fileId'] = fid # :tada: download the data! :tada: - with requests.get(url, params=params, stream=True) as data: + with requests.get(url, params=params, + headers=headers, stream=True) as data: data.raise_for_status() # construct progress bar @@ -237,7 +239,7 @@ def _download_data(info: Dict[str, Dict[str, str]], fname = data.headers.get('Content-Disposition') fname = re.search('filename="(.+)"', fname).group(1) downloaded.extend([os.path.join(path, fname)]) - with open(downloaded[0], 'wb') as dest: + with open(downloaded[-1], 'wb') as dest: dest.write(out.read()) return downloaded @@ -342,7 +344,7 @@ def fetch_studydata(*datasets: str, info = {dset: _STUDYDATA.get(dset) for dset in datasets} return _download_data(info, url, path=path, user=user, password=password, - overwrite=overwrite, verbose=verbose) + overwrite=overwrite, verbose=verbose, bundle=False) def fetch_genetics(*datasets: str, diff --git a/pypmi/tests/test_fetchers.py b/pypmi/tests/test_fetchers.py index 96d79c1..25df4b3 100644 --- a/pypmi/tests/test_fetchers.py +++ b/pypmi/tests/test_fetchers.py @@ -29,11 +29,11 @@ def test_get_download_params(url): def test_fetchable_studydata(): # 113 datasets available, should get a list of them dsets = fetchers.fetchable_studydata() - assert isinstance(dsets, list) and len(dsets) == 113 + assert isinstance(dsets, list) and len(dsets) == 112 @pytest.mark.parametrize(('datasets', 'expected'), [ - (['all'], 113), + (['all'], 112), (['Code list', 'Clinical labs'], 2) ]) def test_fetch_studydata(studydata, datasets, expected): diff --git a/pypmi/utils.py b/pypmi/utils.py index ab4393e..737f797 100644 --- a/pypmi/utils.py +++ b/pypmi/utils.py @@ -82,6 +82,7 @@ def _get_data_dir(path: str = None, if path is None: try: path = os.environ['PPMI_PATH'] + os.makedirs(path, exist_ok=True) except KeyError: path = os.getcwd()