3
\$\begingroup\$

I am scraping a Wikipedia page, getting info from that page and instantiating a new object with the information collected:

require_relative '../config/environment'

def get_winner
    award = Nokogiri::HTML(open("http://en.wikipedia.org/wiki/Academy_Award_for_Best_Picture"))
    year = nil
    award.css('table.wikitable').each do |table|
        @year = winning_year(table).join("")
        winner_name =  winner_title(table)
        url_extention = table.css('td a').first.attr('href')
        winner_page = Nokogiri::HTML(open("http://en.wikipedia.org#{url_extention}"))
         budget = "N/A"
         winner_page.css('.vevent tr').each do |node|
           if node.text.include?('Budget')
             budget = format_budget(node.css('td').text)
           end
         end
         AcademyAwardBestPicture.new(@year,winner_name,budget)
    end
end


def winning_year(table)
    table.css('caption big').map do |a_tag|
        if a_tag.text.include?('[')
            a_tag.text.split('[').first
        else
            a_tag.text
        end
    end
end

def winner_title(table)
    table.css('td a').first.text
end

def format_budget(budget_money)
    formated_number = nil
    if budget_money.include?("-")
        budget_money = budget_money.split("-").first + " million"
    elsif budget_money.include?("–")
        budget_money = budget_money.split("–").first + " million"
    end
        budget_money.scan(/[\$£](\d{1,3}(,\d{3})*(\.\d*)?)( million)?/).each do |m|
          number = m[0].gsub(',', '').to_f
          unit = m[3]
          multiplier = 1
            if number == 55.0
                multiplier = 1e6
            end
              if unit == ' million'
                multiplier = 1e6
              end
            formated_number = number * multiplier
                if budget_money.include?("£")
                    formated_number = british_pound_currency_convert(formated_number)
                end
            end
        formated_number
end

def british_pound_currency_convert(amount)
    web_page = Nokogiri::HTML(open('http://measuringworth.com/datasets/exchangeglobal/result.php?year_source=1791&year_result=2007&countryE%5B%5D=United+Kingdom'))
        exchange_rate_table = web_page.css('table').last
        exchange_rate_cell = exchange_rate_table.css('tr')
        formated_amount = nil
        exchange_rate_cell.each do |cell|
            if @year == cell.children[0].text
                one_dollar_to_pound_rate = cell.children[1].text.gsub(/[^0.-9]/,'').to_f
                formated_amount = (amount / one_dollar_to_pound_rate).round(0)
            elsif @year == "2010"
                exchange_rate_2010 = 1.561
                formated_amount = (amount * exchange_rate_2010).round(0)
            end
        end
        formated_amount
end

The code is pretty slow, which takes about 13 seconds to scrape 87 "award winning best picture". Does anybody have suggestions on how to make it faster/ more robust?

\$\endgroup\$
  • 1
    \$\begingroup\$ Try mobile version of the page. Or google another knowledge base service, which has API. \$\endgroup\$ – Nakilon May 25 '15 at 17:44
  • \$\begingroup\$ Could you give us the file at ../config/environment as well, or at least a link to it? \$\endgroup\$ – Nic Hartley Jul 2 '15 at 17:59

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.