5
\$\begingroup\$

I am using this R function to web scrape data from the google play store. Is there a way to increase its efficiency using R? This code takes about 4 seconds for 14 urls with my machine/internet connection.

library(rvest) 
library(RCurl) 

library(foreach)
library(doMC) 
registerDoMC(4) #insert number of your real Cores


urls<-c("https://play.google.com/store/apps/details?id=com.nippt.somalidictionary.pro", 
        "https://play.google.com/store/apps/details?id=net.bigdict.bdislt.free", 
        "https://play.google.com/store/apps/details?id=com.authorly.piggy", 
        "https://play.google.com/store/apps/details?id=com.gulliverstravelguides.london", 
        "https://play.google.com/store/apps/details?id=com.sms.ghorbhara", 
        "https://play.google.com/store/apps/details?id=com.mamnil.burgercrushhd", 
        "https://play.google.com/store/apps/details?id=com.horo.videos.de.moda", 
        "https://play.google.com/store/apps/details?id=cz.anywhere.autohydraulik", 
        "https://play.google.com/store/apps/details?id=com.skylineapps.mortgage", 
        "https://play.google.com/store/apps/details?id=air.RouletteMultiplay", 
        "https://play.google.com/store/apps/details?id=com.lepassagecarcassonne.lepassagecarcassonne", 
        "https://play.google.com/store/apps/details?id=com.dakshaa.hr", 
        "https://play.google.com/store/apps/details?id=com.touchN.ChineseMyth10google", 
        "https://play.google.com/store/apps/details?id=hali.retrotv" 
)


get.asynch <- function(urls){
    txt <- getURLAsynchronous(urls)
    mclapply(X = txt, function(txt){
        doc <- try(html(txt,encoding = "UTF-8"))
        #Title 
        title<-try({doc %>% html_node(".document-title div") %>% html_text()}, silent=TRUE) 
        #installed base
        instb<-try({doc %>% html_node(".meta-info:nth-child(3) .content") %>% html_text()}, silent=TRUE)
        #Price
        price<-try({doc %>% html_node("#body-content > div > div > div.main-content > div:nth-child(1) > div > div.details-info > div.info-container > div.details-actions > span > span > button > span:nth-child(3)") %>%html_text()}, silent=TRUE)
        #Necessary Version
        nc1<-try({doc %>% html_node("#body-content > div.outer-container > div > div.main-content > div:nth-child(4) > div > div.details-section-contents > div:nth-child(5) > div.content") %>% html_text()}, silent=TRUE)
        #Necessary Version 2
        nc2<-try({doc %>% html_node("#body-content > div.outer-container > div > div.main-content > div:nth-child(3) > div > div.details-section-contents > div:nth-child(5) > div.content") %>% html_text()}, silent=TRUE)
        #Necessary Version 3
        nc3<-try({doc %>% html_node("#body-content > div > div > div.main-content > div:nth-child(2) > div > div.details-section-contents > div:nth-child(5) > div.content") %>% html_text()}, silent=TRUE)
        #Number Reviews
        nr<-try({doc %>% html_node(".reviews-num") %>% html_text() %>% as.numeric()}, silent=TRUE)
        #five Star reviews
        fs<-try({doc %>% html_node(".five .bar-number") %>%html_text() %>%as.numeric()}, silent=TRUE)
        #four Star reviews
        fos<-try({doc %>% html_node(".four .bar-number") %>%html_text() %>%as.numeric()}, silent=TRUE)
        #three Star reviews
        ts<-try({doc %>% html_node(".three .bar-number") %>%html_text() %>%as.numeric()}, silent=TRUE)
        #two star reviews
        tws<-try({doc %>% html_node(".two .bar-number") %>%html_text() %>%as.numeric()}, silent=TRUE)
        #one Star review
        os<-try({doc %>% html_node(".one .bar-number") %>%html_text() %>%as.numeric()}, silent=TRUE)
        #category
        category1<-try({doc %>% html_node(".category span") %>%html_text() }, silent=TRUE)
        #age restric
        age.res<-try({doc %>%html_node(".meta-info.contains-text-link div.content") %>%html_text() }, silent=TRUE)
        #developer
        dev<-try({doc %>% html_node("#body-content > div.outer-container > div > div.main-content > div:nth-child(1) > div > div.details-info > div.info-container > div:nth-child(2) > a > span") %>%html_text()}, silent=TRUE)

        results<-return(list(title, instb, price, nc1,nc2,nc3,nr,fs,fos,ts,tws,os,category1, age.res,dev))
    })
}

system.time(mydata<- data.frame(matrix(unlist(get.asynch(urls)), nrow=length(urls), byrow=T),stringsAsFactors=FALSE))
\$\endgroup\$
  • 1
    \$\begingroup\$ Sounds unlikely. If you could use a profiler, you'd see that most of the time is spent in getURLAsynchronous, i.e. downloading the data. \$\endgroup\$ – flodel Sep 3 '15 at 10:49
  • \$\begingroup\$ Try download URLs with the mclapply too. \$\endgroup\$ – Artem Klevtsov Jan 8 '16 at 20:39
  • \$\begingroup\$ Thanks for the suggestion. but using the above code I am not able to get any output. All cells are coming as "NA". Can you help me in this ? \$\endgroup\$ – Subhajit pal Apr 8 '18 at 11:05
  • \$\begingroup\$ @Subhajitpal I suspect that is because the HTML of the Google Play store has changed in the past two and a half years. \$\endgroup\$ – Simon Forsberg Apr 8 '18 at 11:42

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.