From 9e4969c38af4c2749231e4d00f33d06ee6014f2b Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Wed, 6 Apr 2022 09:31:57 +0200 Subject: [PATCH] Improve download throughput The current version of the download method limits the throughput at a maximum of 40 KB/s (due to chunk size and the sleep). The fix increases the chunk size so 16 KB, factors it out as a class member and remove the sleep in order to allow for maximum possible throughput --- ssgetpy/matrix.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ssgetpy/matrix.py b/ssgetpy/matrix.py index 97537fd..a5b8f09 100644 --- a/ssgetpy/matrix.py +++ b/ssgetpy/matrix.py @@ -39,6 +39,7 @@ class Matrix: `isspd` : True if this matrix is symmetric, positive definite `kind` : The underlying problem domain """ + CHUNK_SIZE = 16384 attr_list = [ "Id", @@ -203,10 +204,9 @@ def download(self, format="MM", destpath=None, extract=False): with open(localdest, "wb") as outfile, tqdm( total=content_length, desc=self.name, unit="B" ) as pbar: - for chunk in response.iter_content(chunk_size=4096): + for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE): outfile.write(chunk) - pbar.update(4096) - time.sleep(0.1) + pbar.update(self.CHUNK_SIZE) if extract and (format == "MM" or format == "RB"): bundle.extract(localdest)