Below is my code to download file as fast as possible using asycio Trying to implement, multiple server connection with multi chunk, multi threded downlad just like idm and aria2
import asyncio
import os.path
import shutil
import aiofiles
import aiohttp
import lxml.html as htmlparser
import cssselect
import regex, json
from tempfile import TemporaryDirectory
domain = "https://domain.com/"
url = 'https://domain.com/ust/xxxx'
CONTENT_ID = regex.compile(r"/ust/([^?#&/]+)")
def parts_generator(size, start=0, part_size=5 * 1024 ** 2):
while size - start > part_size:
yield start, start + part_size
start += part_size
yield start, size
async def main():
async def download(url, headers, save_path):
async with session.get(url, headers=headers) as request:
file = await aiofiles.open(save_path, 'wb')
await file.write(await request.content.read())
async with aiohttp.ClientSession() as session:
async with session.get(url) as first:
cs = await first.text()
csrf_token = htmlparser.fromstring(cs).cssselect("meta[name='csrf-token']")[0].get("content")
content_id = CONTENT_ID.search(url).group(1)
Headers={"x-requested-with": "XMLHttpRequest", "x-csrf-token": csrf_token}
async with session.post(domain + "api/get&user=xxx&pass=yyy", headers=Headers, json={"id": content_id}) as resp:
res = json.loads(await resp.text())
re = res['result']['Original']['file']
async with session.get(re) as request:
size = request.content_length
tasks = []
file_parts = []
filename = 'File.mp4'
tmp_dir = TemporaryDirectory(prefix=filename, dir=os.path.abspath('.'))
for number, sizes in enumerate(parts_generator(size)):
part_file_name = os.path.join(tmp_dir.name, f'{filename}.part{number}')
file_parts.append(part_file_name)
tasks.append(await download(re, {'Range': f'bytes={sizes[0]}-{sizes[1]}'}, part_file_name))
await asyncio.gather(*tasks)''
with open(filename, 'wb') as wfd:
for f in file_parts:
with open(f, 'rb') as fd:
shutil.copyfileobj(fd, wfd)
asyncio.run(main())
Can anyone help improvise it. is using asyncio better than threadpool and multiprocessing
my script still doesnt perform concurrent download
help with adding concurrency and handling cases where server repond with empty payload in casr of excessive request. ysing true loop to sleep and retry in such case
with idm this mp4 700mb can be downloaded within few minutes and idm can achive 3 mbps speed for this video download on my network
can someone help tweak my python script to achive same spped and failsafe downlaod like idm Also want to be able to play file simultaneiusly while downlaoding