Uncaught network problem exceptions in CLI dataset download #94
Description
Here's the output I once got:
$ atldld dataset download 75492803 out
Dataset ID : 75492803
Downsample reference : 25
Downsample image : 0
Include expression : False
Output folder : out
Number of section images : 20
Section thickness : 25.0µm
Plane of section : 2
[###---------------------------------] 10% 00:00:45
Traceback (most recent call last):
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/urllib3/response.py", line 438, in _error_catcher
yield
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/urllib3/response.py", line 767, in read_chunked
chunk = self._handle_chunk(amt)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/urllib3/response.py", line 720, in _handle_chunk
returned_chunk = self._fp._safe_read(self.chunk_left)
File "/Users/sschmidt/.pyenv/versions/3.8.10/lib/python3.8/http/client.py", line 611, in _safe_read
raise IncompleteRead(data, amt-len(data))
http.client.IncompleteRead: IncompleteRead(874 bytes read, 7126 more expected)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/models.py", line 753, in generate
for chunk in self.raw.stream(chunk_size, decode_content=True):
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/urllib3/response.py", line 572, in stream
for line in self.read_chunked(amt, decode_content=decode_content):
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/urllib3/response.py", line 793, in read_chunked
self._original_response.close()
File "/Users/sschmidt/.pyenv/versions/3.8.10/lib/python3.8/contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/urllib3/response.py", line 455, in _error_catcher
raise ProtocolError("Connection broken: %r" % e, e)
urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(874 bytes read, 7126 more expected)', IncompleteRead(874 bytes read, 7126 more expected))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/bin/atldld", line 33, in <module>
sys.exit(load_entry_point('atldld', 'console_scripts', 'atldld')())
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/click/core.py", line 1128, in __call__
return self.main(*args, **kwargs)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/click/core.py", line 1053, in main
rv = self.invoke(ctx)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/click/core.py", line 1659, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/click/core.py", line 1659, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/click/core.py", line 1395, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/click/core.py", line 754, in invoke
return __callback(*args, **kwargs)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/src/atldld/cli/dataset.py", line 201, in dataset_download
for image_id, section_coordinate, img, img_expr, df in progress:
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/click/_termui_impl.py", line 328, in generator
for rv in self.iter:
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/src/atldld/sync.py", line 420, in run
img = get_image(
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/src/atldld/utils.py", line 122, in get_image
response = requests.get(url)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/sessions.py", line 677, in send
history = [resp for resp in gen]
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/sessions.py", line 677, in <listcomp>
history = [resp for resp in gen]
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/sessions.py", line 237, in resolve_redirects
resp = self.send(
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/sessions.py", line 697, in send
r.content
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/models.py", line 831, in content
self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b''
File "/Users/sschmidt/Code/BlueBrain/atlas-download-tools/venv/lib/python3.8/site-packages/requests/models.py", line 756, in generate
raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: IncompleteRead(874 bytes read, 7126 more expected)', IncompleteRead(874 bytes read, 7126 more expected))It would be better to catch connection problems, print a useful error message, e.g. try again, and exit with an error code.