You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Not sure if I'm somehow doing something wrong, but I am not able to get RunUseCase.py to download data. My authentication with the ShapeWorks data portal works fine, I just always get a ConnectionResetError before the data is finished downloading.
Here is my output:
C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\python.exe "C:\Users\acre018\OneDrive - The University of Auckland\Documents\Python Projects\Shapeworks Examples\Python\RunUseCase.py" ellipsoid_fd --tiny_test
Using shapeworks module from C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\shapeworks\__init__.py
Using shapeworks from C:\Program Files\ShapeWorks\bin\shapeworks.EXE
Step 1. Acquire Data
_____ ___ .
| | / \ / \
| ShapeWorks Cloud \
|_____| \___/ /_____\
Login to ShapeWorks Data Portal successful
Downloading files to Output/ellipsoid_fd/ellipsoid_fd
Downloading 96 files...
Traceback (most recent call last):-------| 3.1% Complete.
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connectionpool.py", line 403, in _make_request
self._validate_conn(conn)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connectionpool.py", line 1053, in _validate_conn
conn.connect()
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connection.py", line 419, in connect
self.sock = ssl_wrap_socket(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\util\ssl_.py", line 449, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\util\ssl_.py", line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\ssl.py", line 501, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\ssl.py", line 1041, in _create
self.do_handshake()
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\ssl.py", line 1310, in do_handshake
self._sslobj.do_handshake()
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\requests\adapters.py", line 440, in send
resp = conn.urlopen(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connectionpool.py", line 798, in urlopen
retries = retries.increment(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\util\retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\packages\six.py", line 769, in reraise
raise value.with_traceback(tb)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connectionpool.py", line 403, in _make_request
self._validate_conn(conn)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connectionpool.py", line 1053, in _validate_conn
conn.connect()
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\connection.py", line 419, in connect
self.sock = ssl_wrap_socket(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\util\ssl_.py", line 449, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\urllib3\util\ssl_.py", line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\ssl.py", line 501, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\ssl.py", line 1041, in _create
self.do_handshake()
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\ssl.py", line 1310, in do_handshake
self._sslobj.do_handshake()
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\acre018\OneDrive - The University of Auckland\Documents\Python Projects\Shapeworks Examples\Python\RunUseCase.py", line 89, in <module>
module.Run_Pipeline(args)
File "C:\Users\acre018\OneDrive - The University of Auckland\Documents\Python Projects\Shapeworks Examples\Python\ellipsoid_fd.py", line 30, in Run_Pipeline
sw.portal.download_dataset(dataset_name, output_directory)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\shapeworks\portal.py", line 106, in download_dataset
project.download(Path(download_path))
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\swcc\models\project.py", line 374, in download
file_item.download(Path(folder, *path.split('/')[:-1]))
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\swcc\models\file_type.py", line 117, in download
r = requests.get(self.url, stream=True)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\requests\api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\requests\sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\requests\sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "C:\Users\acre018\AppData\Local\miniconda3\envs\shapeworks\lib\site-packages\requests\adapters.py", line 501, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
Process finished with exit code 1
The text was updated successfully, but these errors were encountered:
I'll try to test from my home network / a vpn this evening. In the meantime, could I suggest adding a few retries to the file downloads? Something like this worked for me (in the Project class of swcc.models.project):
def download_with_retries(self, item, *args, retries=100):
for i in range(retries+1):
try:
item.download(*args)
return
except requests.exceptions.ConnectionError:
if i == retries:
raise
print(f"Error downloading {item}, retry {i+1}... ")
def download(self, folder: Union[Path, str]):
session = current_session()
self.download_with_retries(self.file, folder)
r: requests.Response = session.get(f'{self._endpoint}/{self.id}/download/')
raise_for_status(r)
data = r.json()
files = data['download_paths']
print(f'Downloading {len(files)} files...')
print_progress_bar(0, len(files))
for index, (path, url) in enumerate(files.items()):
file_item: FileType = FileType(url=url)
self.download_with_retries(file_item, Path(folder, *path.split('/')[:-1]))
print_progress_bar(index + 1, len(files))
session.close()
print()
Not sure if I'm somehow doing something wrong, but I am not able to get RunUseCase.py to download data. My authentication with the ShapeWorks data portal works fine, I just always get a ConnectionResetError before the data is finished downloading.
Here is my output:
The text was updated successfully, but these errors were encountered: