Python multi-connection downloader

Question

This is a simple Python multi-connection downloader primarily using requests, mmap and threads, it downloads a single file using 32 concurrent connections, slices the download using range parameter, and write the slices to a mmap object.

The code:

import re
import requests
import sys
import time
from collections import deque
from datetime import datetime, timedelta
from math import inf
from mmap import mmap
from pathlib import Path
from reprint import output
from threading import Thread

def timestring(sec):
    sec = int(sec)
    a = str(int(sec // 3600)).zfill(2)
    sec = sec % 3600
    b = str(int(sec // 60)).zfill(2)
    c = str(int(sec % 60)).zfill(2)
    return '{0}:{1}:{2}'.format(a, b, c)

class downloader:
    def __init__(self, url, filepath, num_connections=32, overwrite=False):
        self.mm = None
        self.count = 0
        self.recent = deque([0] * 20, maxlen=20)
        self.download(url, filepath, num_connections, overwrite)
    
    def multidown(self, url, start, end):
        r = requests.get(url, headers={'range': 'bytes={0}-{1}'.format(start, end-1)}, stream=True)
        i = start
        for chunk in r.iter_content(1048576):
            if chunk:
                self.mm[i: i+len(chunk)] = chunk
                self.count += len(chunk)
                i += len(chunk)
    
    def singledown(self, url, path):
        with requests.get(url, stream=True) as r:
            with path.open('wb') as file:
                for chunk in r.iter_content(1048576):
                        if chunk:
                            self.count += len(chunk)
                            file.write(chunk)
    
    def download(self, url, filepath, num_connections=32, overwrite=False):
        singlethread = False
        threads = []
        bcontinue = False
        filepath = filepath.replace('\\', '/')
        if (not re.match('^[a-zA-Z]:/(((?![<>:"/|?*]).)+((?<![ .])/)?)*$', filepath) or 
            not Path(filepath[:3]).exists()):
            print('Invalid windows file path has been inputted, process will now stop.')
            return
        path = Path(filepath)
        if not path.exists():
            bcontinue = True
        else:
            if path.is_file():
                if overwrite:
                    bcontinue = True
                else:
                    while True:
                        answer = input(f'`{filepath}` already exists, do you want to overwrite it? \n(Yes, No):').lower()
                        if answer in ['y', 'yes', 'n', 'no']:
                            if answer.startswith('y'):
                                bcontinue = True
                            break
                        else:
                            print('Invalid input detected, retaking input.')
        if not bcontinue:
            print(f'Overwritting {filepath} has been aborted, process will now stop.')
            return
        bcontinue = False
        head = requests.head(url)
        if head.status_code == 200:
            bcontinue = True
        else:
            for i in range(5):
                print(f'Failed to connect server, retrying {i + 1} out of 5')
                head = requests.head(url)
                if head.status_code == 200:
                    print(f'Connection successful on retry {i + 1}, process will now continue.')
                    bcontinue = True
                    break
                else:
                    print(f'Retry {i + 1} out of 5 failed to connect, reattempting in 1 second.')
                    time.sleep(1)
        if not bcontinue:
            print("Connection can't be established, can't download target file, process will now stop.")
            return
        folder = '/'.join(filepath.split('/')[:-1])
        Path(folder).mkdir(parents=True, exist_ok=True)
        headers = head.headers
        total = headers.get('content-length')
        if not total:
            print(f'Cannot find the total length of the content of {url}, the file will be downloaded using a single thread.')
            started = datetime.now()
            print('Task started on %s.' % started.strftime('%Y-%m-%d %H:%M:%S'))
            th = Thread(target=self.singledown, args=(url, path))
            threads.append(th)
            th.start()
            total = inf
            singlethread = True
        else:
            total = int(total)
            code = requests.head(url, headers={'range':'bytes=0-100'}).status_code
            if code != 206:
                print('Server does not support the `range` parameter, the file will be downloaded using a single thread.')
                started = datetime.now()
                print('Task started on %s.' % started.strftime('%Y-%m-%d %H:%M:%S'))
                th = Thread(target=self.singledown, args=(url, path))
                threads.append(th)
                th.start()
                singlethread = True
            else:
                path.touch()
                file = path.open(mode='wb')
                file.seek(total - 1)
                file.write(b'\0')
                file.close()
                file = path.open(mode='r+b')
                self.mm = mmap(file.fileno(), 0)
                segment = total / num_connections
                started = datetime.now()
                print('Task started on %s.' % started.strftime('%Y-%m-%d %H:%M:%S'))
                for i in range(num_connections):
                    th = Thread(target=self.multidown, args=(url, int(segment * i), int(segment * (i + 1))))
                    threads.append(th)
                    th.start()
        downloaded = 0
        totalMiB = total / 1048576
        speeds = []
        interval = 0.025
        with output(initial_len=4, interval=0) as dynamic_print:
            while True:
                status = sum([i.is_alive() for i in threads])
                downloaded = self.count
                self.recent.append(downloaded)
                done = int(100 * downloaded / total)
                doneMiB = downloaded / 1048576
                gt0 = len([i for i in self.recent if i])
                if not gt0:
                    speed = 0
                else:
                    recent = list(self.recent)[20 - gt0:]
                    if len(recent) == 1:
                        speed = recent[0] / 1048576 / interval
                    else:
                        diff = [b - a for a, b in zip(recent, recent[1:])]
                        speed = sum(diff) / len(diff) / 1048576 / interval
                speeds.append(speed)
                nzspeeds = [i for i in speeds if i]
                if nzspeeds:
                    minspeed = min(nzspeeds)
                else:
                    minspeed = 0
                maxspeed = max(speeds)
                meanspeed = sum(speeds) / len(speeds)
                remaining = totalMiB - doneMiB
                dynamic_print[0] = '[{0}{1}] {2}'.format(
                    '\u2588' * done, '\u00b7' * (100-done), str(done)) + '% completed'
                dynamic_print[1] = '{0:.2f} MiB downloaded, {1:.2f} MiB total, {2:.2f} MiB remaining, download speed: {3:.2f} MiB/s'.format(
                    doneMiB, totalMiB, remaining, speed)
                now = datetime.now()
                elapsed = timestring((now - started).seconds)
                if meanspeed and total != inf:
                    eta = timestring(remaining / meanspeed)
                else:
                    eta = '99:59:59'
                dynamic_print[2] = 'Minimum speed: {0:.2f} MiB/s, average speed: {1:.2f} MiB/s, maximum speed: {2:.2f} MiB/s'.format(minspeed, meanspeed, maxspeed)
                dynamic_print[3] = 'Task started on {0}, {1} elapsed, ETA: {2}'.format(
                    started.strftime('%Y-%m-%d %H:%M:%S'), elapsed, eta)
                if status == 0:
                    ended = datetime.now()
                    if not singlethread:
                        self.mm.close()
                    break
                time.sleep(interval)
        time_spent = (ended - started).seconds
        meanspeed = sum(speeds) / len(speeds)
        print('Task completed on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
            ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))

if __name__ == '__main__':
    d = downloader(*sys.argv[1:])

Example usage:

PS C:\Windows\System32> downloader (getdownlink 19711382) "D:/Music/Vox Angeli/Irlande/Vox Angeli - New Soul.mp3"
D:/Music/Vox Angeli/Irlande/Vox Angeli - New Soul.mp3 already exists, do you want to overwrite it?
(Yes, No):y
Task started on 2021-08-07 15:17:18.
[████████████████████████████████████████████████████████████████████████████████████████████████████] 100% completed
3.85 MiB downloaded, 3.85 MiB total, 0.00 MiB remaining, download speed: 2.32 MiB/s
Minimum speed: 0.00 MiB/s, mean speed: 14.01 MiB/s, maximum speed: 52.98 MiB/s
Task started on 2021-08-07 15:17:18, 00:00:00 elapsed, ETA: 00:00:00
Task completed on 2021-08-07 15:17:19, total time elapsed: 00:00:00, mean speed: 14.01 MiB/s

(getdownlink is a Python file that is out of the scope of this review)

The speeds are really high, are the numbers valid? I am using a 100Mbps broadband connection which roughly translates to 11.92MiB/s max download speed, I am not sure if I should trust the numbers, but what is written in the code tells me it is correct.

I want to know whether my code is performant or not, how it can be faster, is 32 connections per download a good practice (I used the limit I have found in most downloaders), and most importantly, I use mmap in this script, I wonder if mmap uses 1MiB physical primary memory for 1MiB of file, in other words, I have 16GiB physical RAM, can I use the same method to download a single file larger than 16GiB?

And I wonder, how can I implement a pause and resume feature, how can I use multithreading to download n (say 4) files simultaneously over a list of files, and how can I display download information for each download?

---

Update:

I have modified my code to allow it download large files, and I have determined that mmap doesn't use 1MiB physical memory per 1MiB file, as evident by my testing.

The following demonstrates usage (I don't know how to pass default parameters to script yet, and the link might become expired):

PS C:\Windows\System32> downloader "http://51.195.5.190/Oceanofgames.com/Running_With_Rifles_Edelweiss_PLAZA.zip?md5=i-0EWKAFXjhhWvF17S8j3A&expires=1630915755" 'D:\Downloads\Running with rifles.zip'
D:/Downloads/Running with rifles.zip already exists, do you want to overwrite it?
(Yes, No):y
Task started on 2021-08-07 16:55:57.
[████████████████████████████████████████████████████████████████████████████████████████████████████] 100% completed
1489.83 MiB downloaded, 1489.83 MiB total, 0.00 MiB remaining, download speed: 22.29 MiB/s
Minimum speed: 0.00 MiB/s, mean speed: 11.66 MiB/s, maximum speed: 160.00 MiB/s
Task started on 2021-08-07 16:55:57, 00:03:04 elapsed, ETA: 00:00:00
Task completed on 2021-08-07 16:59:02, total time elapsed: 00:03:04, mean speed: 11.66 MiB/s

While it is downloading, the download speed constantly shifts between 0 and a non-zero number, how can I get current download speed?

---

Minor update: made several small improvements, and made the dummy ETA more logical.

---

Major update: used a fixed size deque to keep track of recent sizes within 0.5 second time frame, and use that data to calculate current download speed.

---

Minor update: Updated the minimum speed logic so that if there are non zero speeds the minimum speed will be the minimum of non-zero speeds instead of 0.

---

Final update:

Updated the code so that single thread downloads will also show additional information, and limit the RAM single thread downloading can use.

Answer 1

Here are a few comments on your question and your code, in no particular order:

• There is no need to reinvent the wheel when formatting a time (timestring function). Have a look at datetime.date#strftime.

• Instead of handling the threads yourself, you could probably use a ThreadPool and let it handle the actual thread management.

• A constructor is expected to only create an instance of a class. Having the constructor of downloader do the actual download is a bit obscure and nasty. An instance should first be created, and the explicitly call a download method.

• The cyclomatic complexity of your download function is about 35. Although the recommended cyclomatic complexity of a method is subject to opinion, I'd say if it's over 10, ot should probably be refactored.

This only means you should separate concerns and levels of abstraction. For example, this piece of code could go into its own method:

filepath = filepath.replace('\\', '/')
if (not re.match('^[a-zA-Z]:/(((?![<>:"/|?*]).)+((?<![ .])/)?)*$', filepath) 
        or not Path(filepath[:3]).exists()):
    print('Invalid windows file path has been inputted, process will now stop.')
    return
path = Path(filepath)

• If you haven't yet, have a look at PEP-8, a general style guide for python code widely followed by the community. For example, class names should start with a capital letter: Downloader instead of downloader.

• You should consider using type hints in your methods, to make them easier to understand. For example:

def timestring(sec: int) -> str:
  ...