2
\$\begingroup\$

Below is a python script which scrapes specific google apps url for example https://play.google.com/store/apps/details?id=com.mojang.minecraftpe and save the developer email,application name and price in mysql database with database name "app_database" and inside table named "app_table". There are lot of data type conversions and workarounds here and i think that might not come under best practices. How can I improve this.

from lxml import html
import requests
import MySQLdb
conn = MySQLdb.connect("localhost","root","pA$$W0rd","app_database")
cursor=conn.cursor()
class AppCrawler:

    def __init__(self,starting_url,depth):
        self.starting_url=starting_url
        self.depth=depth
        self.current_depth=0
        self.depth_links=[]
        self.apps=[]

    def crawl(self):
        app=self.get_app_from_link(self.starting_url)
        self.apps.append(app)
        self.depth_links.append(app.links)

        while self.current_depth < self.depth:      #0 < 0
            current_links=[]
            for link in self.depth_links[self.current_depth]:
                current_app = self.get_app_from_link(link)
                current_links.extend(current_app.links)
                self.apps.append(current_app)
            self.current_depth += 1
            self.depth_links.append(current_links)

        return

    def get_app_from_link(self,link):
        start_page=requests.get(link)
        #print start_page.text
        tree = html.fromstring(start_page.text)
        # the tree.xpath(...) gives an array 
        name = tree.xpath('//div[@class="id-app-title"]/text()')[0]  

        #developer=tree.xpath('//a[@class="dev-link"]//*/div/@href')
        temp_addr=tree.xpath('//a[contains(@href,"mailto") and @class="dev-link"]/@href')[0]
        developer=temp_addr.split(':')

        price=tree.xpath('//button[@class="price buy id-track-click id-track-impression"]/span[2]/text()')
        temp_price=price[0].split()
        price=str(temp_price[0].lstrip('$'))
        if price=="Install":
            price="0"
        templinks = tree.xpath("//a[@class='title']/@href")

        links=[]
        length=len(templinks)
        domain="https://play.google.com"
        for i in templinks:
            i="https://play.google.com"+i
            links.append(i)
#        print links            
        #print len(testlinks)
        #links = [h for h in tree.xpath("//a[@class='title']/@href")]
        #print links
        #return name,developer[1],price,links
        app = App(name,developer[1],price,links)
        return app

class App:
    def __init__(self, name,developer,price,links):
        self.name=name
        self.developer=developer
        self.price = price
        self.links=links

    def __str__(self):
        return(self.name.encode('UTF-8')+"|"+
        self.developer.encode('UTF-8')+"|"+
        self.price.encode('UTF-8'))


crawler = AppCrawler('https://play.google.com/store/apps/details?id=com.mojang.minecraftpe',0)
crawler.crawl()
#print type(crawler.apps)

query="INSERT INTO app_table (name,developer,price) values('%s','%s','%d ')"
for app in crawler.apps:
    data=str(app).split('|',2)
    price=float(data[2])
    try:
        cursor.execute(query % (data[0],data[1],price))
        print "OK"
        conn.commit()
    #row=x.fetcha
    except:
        print "Failed"
        conn.rollback()
    print data[0],data[1],price
\$\endgroup\$
0
\$\begingroup\$

This is being ported over to python 3.x from python 2.x

  • Docstrings: You should include a docstring at the beginning of every method/class/module you write. This will allow documentation to identify what your code is supposed to do. PEP-8 Docstring Conventions.
  • Variable / Operator Spacing: When not in default parameters, you should put spaces between variables, the =, and the content it's being assigned. Also, there should be spaces between variables that are being passed to methods. PEP-8 Whitespace Conventions.
  • Import Ordering: Standard library imports should be put first, then related third party imports, then local libraries/files, all grouped and separated by a blank line. PEP-8 Import Conventions.
  • String Formatting: Your __str__ in your App class is a bit messy. If you see this behavior, something is afoot. This is a red flag to me. I use f"" to directly include variables into my strings, instead of having to type cast them or, in your case, having to use multiple + to concatenate the variables and the strings.
  • Unnecessary return statements: Your return in your crawl method is unnecessary. Python automatically returns at the end of functions, so having that return isn't needed.

Updated Code

"""
Module Docstring:
A description of your program goes here
"""

import requests
import MySQLdb

from lxml import html

class AppCrawler:
    """
    Class for crawling a link, and gathering information about an app
    """
    def __init__(self, starting_url, depth):
        """
        AppCrawler Class Init Method

        :param starting_url: The starting url for the crawler
        :param depth: How deep the app crawler should go

        """
        self.starting_url = starting_url
        self.depth = depth
        self.current_dept = 0
        self.depth_links = []
        self.apps = []

    def crawl(self):
        """
        Crawls to find links connected to apps
        """
        app = self.get_app_from_link(self.starting_url)
        self.apps.append(app)
        self.depth_links.append(app.links)

        while self.current_depth < self.depth: #0 < 0
            current_links = []
            for link in self.depth_links[self.current_depth]:
                current_app = self.get_app_from_link(link)
                current_links.extend(current_app.links)
                self.apps.append(current_app)
            self.current_depth += 1
            self.depth_links.append(current_links)

    def get_app_from_link(self, link):
        """
        Gets the app from the passed `link`

        :param link: Link to the app

        """
        start_page = requests.get(link)

        tree = html.fromstring(start_page.text)

        name = tree.xpath('//div[@class="id-app-title"]/text()')[0]  

        temp_addr = tree.xpath('//a[contains(@href,"mailto") and @class="dev-link"]/@href')[0]
        developer = temp_addr.split(':')

        price = tree.xpath('//button[@class="price buy id-track-click id-track-impression"]/span[2]/text()')
        temp_price = price[0].split()
        price = str(temp_price[0].lstrip('$'))
        if price == "Install":
            price = "0"
        templinks = tree.xpath("//a[@class='title']/@href")

        links = []
        length = len(templinks)
        domain = "https://play.google.com"
        for i in templinks:
            i = f"https://play.google.com{i}"
            links.append(i)
        app = App(name, developer[1], price, links)
        return app

class App:
    """
    Class to represent each app getting crawled
    """
    def __init__(self, name, developer, price, links):
        """
        App Class Init Method

        :param name: Name of the app
        :param developer: Developer of the app
        :param price: Price of the app
        :param links: list of links to the app

        """
        self.name = name
        self.developer = developer
        self.price = price
        self.links = links

    def __str__(self):
        """
        Returns a string representation of the app
        """
        return f"{self.name.encode('UTF-8')}|{self.developer.encode('UTF-8')}|{self.price.encode('UTF-8')}"

if __name__ == '__main__':

    conn = MySQLdb.connect("localhost","root","pA$$W0rd","app_database")
    cursor = conn.cursor()

    crawler = AppCrawler('https://play.google.com/store/apps/details?id=com.mojang.minecraftpe', 0)
    crawler.crawl()

    query = "INSERT INTO app_table (name,developer,price) values('%s','%s','%d ')"
    for app in crawler.apps:
        data = str(app).split('|', 2)
        price = float(data[2])
        try:
            cursor.execute(query % (data[0], data[1], price))
            print("OK")
            conn.commit()
        except:
            print("Failed")
            conn.rollback()
        print(data[0], data[1], price)

| improve this answer | |
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.