1
\$\begingroup\$

This is a simple toy downloader using python's requests library.

I’ve monitored the download process is slower on an ethernet connected box. The realtime speed is measured both on stdout, using get_net_speed() function, and conky.

I’m using a 4Mbps connection.

When downloading a file using wget, the downloading gets full bandwidth as expected. But when using my own downloader script, it gets highest 1Mbps speed.

Here's the code. Review and optimisation are warmly welcome.

#!/usr/bin/env python3

### get details of a requested file and download if a media one 


import requests
import sys
from urllib.parse import urlparse
import socket
import re 
import time
import psutil 

#===================== Main Routine ===================================



def down_in_chunks(resp,file_name,conSize):
    downloaded = 0
    file_descriptor = open(file_name,'wb')
    chunk_size = 1024 * 1024 # 1MB
    for chunk in resp.iter_content(chunk_size=chunk_size):
        downloaded += int(len(chunk)) # get update how much downloaded
        per_cent = 100 * downloaded / int(conSize)
        speed = get_net_speed()
        print("downloading %.2f pc speed %s" %(per_cent,speed),end='\r')
        file_descriptor.write(chunk)
    file_descriptor.close()

#================== Auxilary Routines ==================================

def get_net_speed():
    rcv = psutil.net_io_counters()
    sp0 = rcv[1]
    time.sleep(1)
    rcv = psutil.net_io_counters()
    sp1 = rcv[1]
    speed = sp1 - sp0 
    return convert_size(speed)

def convert_size(file_size):
    for unit in ["B","KiB","MiB","GiB","Tib","PiB","EiB","Zib"]:
        if abs(file_size) <= 1024.0 or abs(file_size) == 0:
            return "%3.1f%s" %(file_size,unit) 
        else:
            file_size /= 1024.0
        #return "%3.1f%s" %(file_size,unit) 


def get_file_details(resp,url):

    file_type = resp.headers["Content-Type"]

    if re.match(r'^html|text',file_type): # perhaps we're not interested in the page only 
        print("header info... ", resp.headers)

    else:
        file_name = url.split("/")[-1]
        print("file name: %s" % file_name)
        conDate = resp.headers["Date"]
        conType = resp.headers["Content-Type"]
        conTn = resp.headers["Connection"]
        conSize = resp.headers["Content-Length"]
        conUnit = resp.headers["Accept-Ranges"]
        humane = convert_size(int(conSize))
        print("time: %s type: %s size: %s (%s) unit: %s connection: %s " %(conDate,conType,conSize,humane,conUnit,conTn))
        return file_name , conSize

# ========== Action ===========

try:

    url = sys.argv[1] # get the url
    urlObj = urlparse(url)
    addr = urlObj.netloc
    ip = socket.gethostbyname(addr.split(":")[0]) # get the ip address
    print("connecting to %s | %s |... " %(addr,ip))
    user_agent = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; hu-HU; rv:1.7.8) Gecko/20050511 Firefox/1.0.4'}
    resp = requests.get(sys.argv[1],stream=True,timeout=15,headers=user_agent) # I'm not a bot Maa'm
    print("status: %s " %(resp.status_code))
    file_name,file_size = get_file_details(resp,url)

    print("downloading.......... %s" %file_name)
    start = time.time()
    down_in_chunks(resp,file_name,file_size)
    end = time.time()
    total_time = end - start
    if total_time >= 60:
        print("download time: %.2f min" %(total_time/60))

    else:
        print("download time: %.2f secs")
    resp.close()


except requests.exceptions.RequestException as E:
    print("Error: ", E)
    sys.exit(0)
\$\endgroup\$
  • \$\begingroup\$ How is get_net_speed measuring the download speed when it pauses the download? \$\endgroup\$ – Graipher Oct 26 '18 at 13:25
  • \$\begingroup\$ Nice point. But needs a concreted explanation \$\endgroup\$ – faquir Oct 26 '18 at 17:47

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.