I recently posted this script:

VIM colors downloader in Python

But since I'm not allowed to update the code there, I wanted to get an idea on this version, that uses multiprocessing:

import os
import json
import requests
from bs4 import BeautifulSoup
from multiprocessing import Process

class Spider:
    def __init__(self, total_pages=40):
        self.base_url = "http://vimcolors.com"
        self.total_pages = total_pages
        self.download_dir = 'colors'
        # If we don't have the download directory
        if not os.path.isdir(self.download_dir):
            print(self.download_dir, 'does not exist, trying to create it...')
            # create it...

    def download(self, name, url):
            full_path = os.path.join(self.download_dir, name)
            # If we have already downloaded this file, just skip
            if os.path.isfile(full_path):
                raise Exception('File: {} already exists; skipping.'.format(name))

            # Get the response
            response = requests.get(url)
            if response.status_code == 404:
                raise Exception('File not found: {}'.format(url))

            # Try downloading the file
            with open(full_path, 'wb') as file_path:
        except Exception as e:
            print('Downloaded', name)

    def crawl(self):
        def repo_formatter(scheme):
            base_url = scheme['github_repo']['address'].replace('github.com', 'raw.githubusercontent.com')
            return '{}/master/colors/'.format(base_url)

        # Loop over all the pages
        for page in range(self.total_pages):
            page_source = requests.get(self.base_url, params={'page': page + 1})
            plain_text = page_source.text
            soup = BeautifulSoup(plain_text, 'lxml')

            # Get the data
            json_data = json.loads(soup.find('div', {'id': 'data'}).attrs['data-colorschemes'])

            # Download the files
            for data in json_data['colorschemes']:
                file_name = data['name'] + '.vim'
                process = Process(target=self.download, args=(file_name, repo_formatter(data),))
                process.daemon = True

        print('\n\nAll done.')

colors_spider = Spider()

Everything is working. I just wanted to know, is multiprocessing a good choice in this case?



Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.