[FIX] Auth update and broken download

.travis.yml

@@ -9,8 +9,8 @@ cache:
   - $HOME/pypmi-data
 
 python:
-  - 3.6
   - 3.7
+  - 3.8
 
 env:
   matrix:
@@ -20,16 +20,16 @@ env:
   global:
     - INSTALL_TYPE=setup
     - PPMI_PATH=$HOME/pypmi-data
-    - secure: "L+LQhRYxaT/0VRnbfxYQ3K/qf//g1kH0Rrn1P1blNFzUUrMkBhsRvusZCkinvze54pi2G6zXs2QvFFSKU5fvkrTiSdfbOiXHBJ5oOOAOG9dehyy2zr2rkzBinDdoEOygiuXQdVKM+JTK1gc3wGASp2TjKbEON341c6jYVIX2SZ8nFfcsfyyNh5Cxf0XC017gZuLi3wqfWFlyyCzbwIPTUq88YI+LroaBmE2C5BtZFC4IHlQsZMzo/SNr8uzJwNXWCi3AGo2QVdgwaHGv4JoJpt+JYlgciFKsIyMkEbZrMLK+wIqtyJdNnode7qPVxzQMVSUaLcHL8usKPmB1Ew5ZSpT0Re257HFsu6HX21tPkkxdztQD0yqMENeM98TSbwEragmrXMl7EjDOEoEzlsZzGVUdGy7Vxz1Ub+pTR/hZRTc0Br4eOhM5AUVrdg/Wl013c14A0y8OOt+xGYvVsUmQX2dgvjhAEXK7AsRwJYCNx5Juuz7gdccF9THO1dGfPcKtGPeFPC5FsxYhYr0ZSzMMDm7/oRRmzxYk2f3w11IlUPGa0sTiZu1BliAVq5r/lhw8UJpB0EgM/UxWUb8bd967//AJM3XqHsdohE+1wyicEfjqPtSnTsvQ5Flrqg3aBxvmcro7fIvKrEJu6ufj6j5mAnGDW/pPBv2zWMCDMvfoDx8="
-    - secure: "IDW2SWqYhT0SFi5cBvh/5NpM8o1S+qJYq0+TASOM4Bs6Gpx5Rd6XWtofr87LaU3RZmwcOng2QvGV7JBZa50f9Fnf3ucNzhX5EfP6h2dS8F52yTh0yoM/TN7Al2rIndYF8YH11o+UjrqoQtDmILZlzwZBQcf/0pIuwcPpZBX21iFnaecZihePYCYvhGgDh0RHaZHEbgtPSMW4gDVrxgWB5hahPRKRofnyfpwGdmbpBbXEHY+Avmumt5hmIpmtagrkz2WHrwq+xr1LCG7MGsrgeDxLQtSwrrgAplCvOtIQW9DDz+avY+LEGFCqt6yd/lzwyzllTZyyRsuWd6oRYBkFVJPR5cDM65NSSIY87ZgfUzQHsBPV9D7vjvfhh5A215LTAIzgZ3JLqT6XUjFvjQ7kn6JwvF5dsnNjc4EV7bz0ru3FtJnCFUJzCUPfZ71hl4CHRF/H74XiPijnpjmzVbdQbk1fm48mrKhG6AmKAUYVMDVd2fVRafM8AgWs2MKkyTmddxufmW4LADhof/T1t9qQZsvfRzJ9WVafPnHtTDVvl0BQO2NG3oV62mCWZNEzdOPojYFEsBXPwUMzh+ekq3+m7K/Bsly/dhT+LYcPwvfqbE4K0/mHVgajF0vO3WoGHnZhbVrz0i51YycAjSdWeI1R3RdCOFMwaBTI5e+0fdjP6M0="
+    - secure: "ZDv6uahvC2HlI5UwpFegljzsUiKqMZY5a8+H0GXhRKgJQXfhjTE+CD0bcCHTq9Z+T1+ZC1mapjBnsS1kxNs3g2WR2z8pVvaWEiqaTxG+YYNRnouxhqhprzzyiMyE1KI0xomHBg56Ph6k4s82v53G3ewuxsujIa9Htu+fmmDFubBhPNRKMud+BGMoTxR0ZNv4q9tBHd4ezYpB8jFyb6RvBpvVTsLrYuHU6KZdVK58EARNgxVSdxYAOSiBGMGD7kvV15eXvaiUHPBLbBK8k0L5CMVeFJHfb+j+sLVgRpmIcKGbCPZMs/cMX0BEQCZ/P+/MScL/7w1+CkLMkkRcZ8be+8yBvCij8U05bXprnwroHupeJaSG18y37o6F19sUevTE0YqfAkIkWeoLipazdzeAe+Mgy34J5onGaE4EdyjPtl9aysQpjc7FyqVRWMIb37wF5F+Apl/w4E3moHy9fv1EajnMZbbznaZbV/LUfonoixFIwMxbcR1p7wEw0gco5NiQJk0Dm+n7W6IHsLiWf8OkZvlQI4Ctlt7q45P+WA5kL3AM27zULEA1Gm2DrV2rlEmDDXrnfxePvdXQo/iDhCpmt1e8lUQMByg/C3BOHpWgpSFGgLFefhlhfxg5MJJL2/0AHwwJvJgtGVbYbSnjdTB2l+WQ/EPFyYYXxwA9idZL49s="
+    - secure: "Nq1Zfdw65IhRhJtWnsUBzn1HcWL4QmxWbafiQI25XkJUg4Ew/W1AaPtdCqR9wW3TRhwDooUtq7a85RrKtnX0Cq97154AF6Nx3pNyLfkgUP6/vVqApOcndFHkApsauFu21gDlbCHd9NokV7CRasiNLaR8jKRnj7c9aGARx2ZvOnK2D9x7wuRwNID8LJzlNtI4QAf0c+2v8YV2dFab08Lh8vNCkH4j3r/Yxy9BXfAZvbYhYWD0MYEEW9fb5i01w0Jd8U/1Ez8i7XPgYP6HgH1erwGRPA+T+fbcUk2cR07FUtucwEFPzcKBs4CxkE9NqgoC/X1fHe+fKzRILxNJ0tXnITeiR/JwgVEaZPL+KrlFKfd2bkjFfLq0wGiKW+g9VoHzKK+E0L0vs0R7+i5JirJq1qy4LnL8dZHS4Iyj4Dz4rQJ5xMruaeHjf1ZU4XZWYiHPq1vo6JtlBPr1so1zPKHIV8Mrph3qZx7I/Qp0hwXY6zJXggaxIGoxwbZjbX7fSNR0QfMyhpDQBSXsBNiTXd8KsX9tjJqOLWkBBfdaLlsfUQqCMJxkIJrmetJ8JZI8nDli41P7QXRond6d4rHmVc5oRgF5Pcfv6kY5ifu44EYfL+tELT26eai59jTGDrpqer/k1FvfgkItYFuyw1XqAZFmMijkqaCxJNZ/NSesZeQbioY="
 
 matrix:
   include:
-    - python: 3.6
+    - python: 3.7
       env:
         - INSTALL_TYPE=sdist
         - CHECK_TYPE=test
-    - python: 3.6
+    - python: 3.7
       env:
         - INSTALL_TYPE=wheel
         - CHECK_TYPE=test
@@ -42,11 +42,12 @@ before_install:
   - if [ "${CHECK_TYPE}" == "test" ]; then
       pip install "pytest>=3.6" pytest-cov coverage coveralls codecov;
     fi
+  - python -m pip install --upgrade numpy
 
 install:
   - |
     if [ "${INSTALL_TYPE}" == "setup" ]; then
-      python setup.py install;
+      pip install ./;
     elif [ "${INSTALL_TYPE}" == "sdist" ]; then
       python setup.py sdist;
       pip install dist/*.tar.gz;

pypmi/data/studydata.json

@@ -27,10 +27,6 @@
         "id": 55,
         "name": "Blood_Chemistry___Hematology.csv"
     },
-    "Center-subject list": {
-        "id": 47,
-        "name": "Center-Subject_List.csv"
-    },
     "Clinical cognitive categorization": {
         "id": 401,
         "name": "Cognitive_Categorization.csv"

pypmi/fetchers.py

@@ -97,8 +97,8 @@ def _download_data(info: Dict[str, Dict[str, str]],
                    user: str = None,
                    password: str = None,
                    overwrite: bool = False,
-                   verbose: bool = True,
-                   bundle: bool = True) -> List[str]:
+                   verbose: bool = False,
+                   bundle: bool = False) -> List[str]:
     """
     Downloads dataset(s) listed in `info` from `url`
 
@@ -190,12 +190,14 @@ def _download_data(info: Dict[str, Dict[str, str]],
 
     # determine whether we're bundling the data (i.e., requesting all files at
     # once) or peforming separate downloads
+    headers = {'Referer': 'https://ida.loni.usc.edu/'}
     if bundle:
         file_ids = [file_ids]
     for fid in file_ids:
         params['fileId'] = fid
         # :tada: download the data! :tada:
-        with requests.get(url, params=params, stream=True) as data:
+        with requests.get(url, params=params,
+                          headers=headers, stream=True) as data:
             data.raise_for_status()
 
             # construct progress bar
@@ -237,7 +239,7 @@ def _download_data(info: Dict[str, Dict[str, str]],
                 fname = data.headers.get('Content-Disposition')
                 fname = re.search('filename="(.+)"', fname).group(1)
                 downloaded.extend([os.path.join(path, fname)])
-                with open(downloaded[0], 'wb') as dest:
+                with open(downloaded[-1], 'wb') as dest:
                     dest.write(out.read())
 
     return downloaded
@@ -342,7 +344,7 @@ def fetch_studydata(*datasets: str,
     info = {dset: _STUDYDATA.get(dset) for dset in datasets}
 
     return _download_data(info, url, path=path, user=user, password=password,
-                          overwrite=overwrite, verbose=verbose)
+                          overwrite=overwrite, verbose=verbose, bundle=False)
 
 
 def fetch_genetics(*datasets: str,

pypmi/tests/test_fetchers.py

@@ -29,11 +29,11 @@ def test_get_download_params(url):
 def test_fetchable_studydata():
     # 113 datasets available, should get a list of them
     dsets = fetchers.fetchable_studydata()
-    assert isinstance(dsets, list) and len(dsets) == 113
+    assert isinstance(dsets, list) and len(dsets) == 112
 
 
 @pytest.mark.parametrize(('datasets', 'expected'), [
-    (['all'], 113),
+    (['all'], 112),
     (['Code list', 'Clinical labs'], 2)
 ])
 def test_fetch_studydata(studydata, datasets, expected):

pypmi/utils.py

@@ -82,6 +82,7 @@ def _get_data_dir(path: str = None,
     if path is None:
         try:
             path = os.environ['PPMI_PATH']
+            os.makedirs(path, exist_ok=True)
         except KeyError:
             path = os.getcwd()