3
\$\begingroup\$

I am trying to process the football tweets and extract information like goals, cards, corners, player name, team name.

I write the code which works, but I may be missing some better python functionalities which can reduce my code or make it better.

# encoding=utf-8
import json
import re, math
import pandas as pd
from nltk import ngrams
from fuzzywuzzy import fuzz
from nltk.stem.lancaster import LancasterStemmer

class NLPExtractor(object):
    def __init__(self):
        self.lancaster_stemmer = LancasterStemmer()
        self.check_words = ["RequestABet", "SkyBet", "point", "rebound", "stage", "btts", "corner", "over", "goal",
                            "yello", "red"]
        self.check_ignore = ["nba", "nfl", "rounds", "knocked", "steals", "rebounds", "draft", "drafted", "to run",
                             "lengths", "promoted", "relegated", "top", "three pointer", "tries", "sets"]
        self.filtered_tweets = []
        with open("football_teams.json") as fp:
            self.football_teams = json.load(fp)
        self.football_teams = [j for i in [self.football_teams[x] for x in self.football_teams] for j in i]
        team_short = ['Hibernian', 'Hibs', 'city', 'Tottenham', 'whu', 'che', 'tot', 'Widnes', 'helens', 'hull', 'cas', 'manu', 'lyon', 'Besiktas', 'Man unt', 'Anderlecht', 'Birmingham', 'Mauritius', 'Seychelles', 'madrid', 'monaco', 'barca', 'juve', 'Barcelona', 'dor', 'Dortmund', 'bar', 'jov', 'mon', 'bayern', 'real', 'Leicester', 'atelti', 'atletico', 'Coventry', 'Munich', 'lei', 'atm', 'rma', 'ath', 'boro', 'sheff u', 'sheff utd', 'bradford', 'Sheffield united', 'United', 'manu', 'utd', 'wba', 'lpool', 'orlando', 'sociedad', 'Man City', 'osa', 'bourn', 'Hurricanes', 'Newcastle', 'leeds', 'Crewe', 'Notts', 'Wilson', 'Featherstone', 'Hull KR', 'Wigan', 'St. Helens', 'Plymouth', 'Shalke', 'Grizzlies', 'Mangotsfield', 'Bristol']
        for name in team_short:
            self.football_teams .append(name)

    def remove_special_chars(self, tokens):
        pattern = "[^a-zA-Z0-9]+"
        filtered_tokens = [re.sub(pattern, "", x) for x in tokens]
        return filtered_tokens

    def get_btts(self, text):
        try:
            if 'no btts' in text:
                btts = 'No'
            elif 'btts' in text:
                btts = 'Yes'
                return btts
            else:
                btts = 'NA'
            return btts
        except:
            pass

    def check_over_under_condition(self, text, score):
        tokens = self.get_ngrams(text, 1)
        filtered_tokens = self.remove_special_chars(tokens)
        ind_score = filtered_tokens.index(score)
        if filtered_tokens[ind_score - 1] == 'over':
            score += 1
            score = math.floor(score)
            return str(score) + ' +'
        elif filtered_tokens[ind_score - 1] == 'under':
            score -= 1
            score = math.ceil(score)
            return str(score) + ' +'
        else:
            return str(score)

    def get_ngrams(self, data, size):
        generator_ngrams = ngrams(str(data).split(), size)
        lst_ngrams = [" ".join(x) for x in generator_ngrams]
        return lst_ngrams

    def filter_ngrams(self, lst_ngrams):
        raw_ngrams = []
        for l in lst_ngrams:
            raw_ngrams.append(re.sub("[^a-zA-Z0-9]+", "", l))
        return raw_ngrams

    def team_names(self, raw_ngrams):
        lst_matched_teams = []
        threshold = 95
        score = -1
        for raw_ngram in raw_ngrams:
            match_team = ""
            for football_team in self.football_teams:
                raw_football_team = re.sub("[^a-zA-Z0-9]+", "", football_team)
                matching_score = fuzz.ratio(raw_football_team.lower(), raw_ngram.lower())
                if score == -1:
                    score = matching_score
                if matching_score >= score and matching_score >= threshold:
                    score = matching_score
                    match_team = football_team
            if match_team:
                lst_matched_teams.append(match_team)
        return lst_matched_teams

    def get_player_name(self, text, check_list):
        import pdb
        # pdb.set_trace()
        player_list = []
        for word in check_list:
             # player_list.append(text.split(word)[0].strip().split(' ')[-1])
            split_text = str(text).split(word)
            if len(split_text) > 1:
                player_list.append(split_text[0].strip().split(' ')[-1])
                try:
                    if  split_text[0].strip().split(' ')[-2] == 'and' or split_text[0].strip().split(' ')[-2] == '&':
                        player_list.append(split_text[0].strip().split(' ')[-3])
                except:
                    print text
                    continue
        return player_list

    def get_player(self, text):
        import pdb
        # pdb.set_trace()
        player_card_check = ['to be booked', 'to be carded']
        player_scorer_check = ['to score']
        player_carded = []
        player_scorer = []
        player_carded = self.get_player_name(text, player_card_check)
        player_scorer = self.get_player_name(text,player_scorer_check)
        return "To carded :" + ", ".join(player_carded) + ", To score : " + ", ".join(player_scorer)     

    def get_match_status(self,text):
        mat_stats = [' draw ', ' lose ', ' win ']
        try:
            for stat in mat_stats:
                status = stat.strip()
                if status in text:
                    return status
        except:
            pass
        return ''

    def get_goals2(self, text):
        try:
            if 'goal' not in text.lower():
                return '0'
            text = text.encode('utf-8').strip()
            data = text.split()
            # import pdb
            # pdb.set_trace()
            for i in range(0, len(data)):
                if re.sub('[^a-zA-Z]', '', data[i].lower()) == 'goals' and len(data[i - 1]) < 4:
                    # print str(data[i - 1]) + ' corners\n'
                    if data[i-2] == 'over':
                        # g = self.remove_special_chars([data[i-1]])[0]
                        goal = str(int(float(data[i-1]))+1) + '+'
                        return goal
                    elif data[i-2] == 'under':
                        # g = self.remove_special_chars([data[i-2]])[0]
                        goal = str(math.ceil(float(data[i-1])-1))
                        return goal
                    else:
                        goal = str(data[i - 1])
                        return goal
                elif re.sub('[^a-zA-Z]', '', data[i].lower()) == 'goals' and len(data[i - 2]) < 4:
                    pdb.set_trace()
                    if data[i-3] == 'over':
                        # g = self.remove_special_chars([data[i-2]])[0]
                        goal = str(int(float(data[i-2]))+1) + '+'
                    elif data[i-3] == 'under':
                        # g = self.remove_special_chars([data[i-2]])[0]
                        goal = str(math.ceil(float(data[i-2])-1))
                    else:
                        goal = data[i-2]
                    # print str(data[i - 2]) + ' corners\n'
                    if data[i-2].lower() == 'match':
                        goal = goal + ' match goals'
                        return goal
                    else:
                        return str(data[i - 2])
            return '0'
        except:
            pass

    def get_corners2(self, text):
        try:
            if 'corner' not in text.lower():
                return '0'
            text = text.encode('utf-8').strip()
            data = text.split()
            # import pdb
            # pdb.set_trace()
            for i in range(0, len(data)):
                if re.sub('[^a-zA-Z]', '', data[i].lower()) == 'corners' and len(data[i - 1]) < 4:
                    # print str(data[i - 1]) + ' corners\n'
                    if data[i-2] == 'over':
                        corn = self.remove_special_chars([data[i-1]])[0]
                        corner = str(int(corn)+1) + '+'
                    else:
                        corner = str(data[i - 1])
                    if data[i-1].lower() == 'match':
                        corner = corner + ' match corners'
                        return corner
                    else:
                        return corner
                elif re.sub('[^a-zA-Z]', '', data[i].lower()) == 'corners' and len(data[i - 2]) < 4:
                    if data[i-3] == 'over':
                        corn = self.remove_special_chars([data[i-2]])[0]
                        corner = str(int(corn)+1)+ '+'
                    else:
                        corner = data[i-2]
                    # print str(data[i - 2]) + ' corners\n'
                    if data[i-1].lower() == 'match':
                        corner = corner + ' match corners'
                        return corner
                    else:
                        return str(data[i - 2])
            return '0'
        except:
            pass

    def get_cards(self, text):
        try:
            text = text.lower()
            if 'card' not in text and 'point' not in text and 'yellow' not in text and 'red' not in text and 'carded' not in text and 'cards' not in text:
                return '0'
            text = text.encode('utf-8').strip()
            data = text.split()
            if 'carded' in text:
                cards = '1+'
                return cards
            else:
                for i in range(0, len(data)):
                    if re.sub('[^a-zA-Z]', '', data[i].lower()) == 'cards' and len(data[i - 1]) < 4:
                        # print str(data[i - 1]) + ' cards\n'
                        cards = str(data[i - 1])
                        d = re.sub('[^0-9]', '', data[i-1])
                        return cards
                    elif re.sub('[^a-zA-Z]', '', data[i].lower()) == 'cards' and len(data[i - 2]) < 4:
                        if data[i-1].lower() == 'match':
                            d = re.sub('[^0-9]', '', data[i-2])
                            cards = str(data[i - 2]) + 'match cards'
                        # print str(data[i - 2]) + ' cards\n'
                        return cards
                    elif re.sub('[^a-zA-Z]', '', data[i].lower()) == 'points' and len(data[i - 1]) < 4 or re.sub('[^a-zA-Z]', '', data[i].lower()) == 'pts' and len(data[i - 1]) < 4:
                        if '+' in data[i-1]:
                            d = re.sub('[^0-9]', '', data[i-1])
                            cards = str(int(d)/10) + '+'
                        else:
                            d = re.sub('[^0-9]', '', data[i-1])
                            cards = str(int(d)/10) 
                        return cards
                    elif re.sub('[^a-zA-Z]', '', data[i].lower()) == 'points' and len(data[i - 2]) < 4 or re.sub('[^a-zA-Z]', '', data[i].lower()) == 'pts' and len(data[i - 2]) < 4:
                        if '+' in data[i-2]:
                            d = re.sub('[^0-9]', '', data[i-2])
                            if data[i-1].lower() == 'match':
                                cards = str(int(d)/10) + '+ match cards' 
                            else:
                                cards = str(int(d)/10) + '+'
                        else:
                            d = re.sub('[^0-9]', '', data[i-2])
                            cards = str(int(d)/10) 
                        return  cards
                    elif re.sub('[^a-zA-Z]', '', data[i].lower()) == 'yellow' and len(data[i - 1]) < 4:
                        cards = str(data[i - 1])
                        return cards
                    elif re.sub('[^a-zA-Z]', '', data[i].lower()) == 'red' and len(data[i - 1]) < 4:
                        cards = str(data[i - 1])
                        return cards
                return '0'
        except:
            pass

    def get_teams(self, text):
        # Bigrams team names
        bigrams = self.get_ngrams(text, 2)
        raw_ngrams = self.filter_ngrams(bigrams)
        bigrams_matched_teams = self.team_names(raw_ngrams)

        # One Gram/ Token team names
        tokens = self.get_ngrams(text, 1)
        raw_ngrams = self.filter_ngrams(tokens)
        onegrams_matched_teams = self.team_names(raw_ngrams)

        # Merging Bi and One ngram team names
        teams = []
        teams = bigrams_matched_teams[:]
        for o in onegrams_matched_teams:
            if bigrams_matched_teams:
                teams.extend([x for x in bigrams_matched_teams if o not in x])
            else:
                teams.append(o)
        teams = list(set(teams))
        return teams

    def get_match_between_teams(self, teams):
        match_between_teams = ""
        if len(teams) == 2:
            match_between_teams = " vs ".join(teams)
        return match_between_teams

    def read_pandas_excel(self):
        res_flag = 0
        dfr = pd.read_excel('Twitter data - #RequestaBet Jan-APR 17.xlsx', sheetname='Sheet1')
        # dfr = pd.read_excel('sample.xlsx', sheetname='Sheet1')
        for i in dfr.index:
            flag = 0
            # text = dfr['Tweet Contents'][i]
            text = dfr['Tweet Contents'][i]
            # text = "@SkyBet hi guys any chance of #RequestABet las palma v Valencia btts & kaka and messi to be booked, rona and gaga to score over 2.5 goals 50+ booking points and 3 corners each".lower()
            try:
                for word in self.check_ignore:
                    if word.lower() in str(text).lower():
                        flag = 1
            except:
                continue
            if flag == 1:
                continue
            teams = self.get_teams(text)
            match_between_teams = self.get_match_between_teams(teams)
            # import pdb
            # pdb.set_trace()
            corners = self.get_corners2(text)
            # corners = self.n_get_corners(text)
            # print "Nitin Corners: ", corners
            cards = self.get_cards(text)
            btts = self.get_btts(text)
            goals = self.get_goals2(text)
            mat_stat = self.get_match_status(text)
            get_player = self.get_player(text)
            # break
            if corners != '0' and cards != '0' and goals != '0' : #and len(match_between_teams)>0
                print text
                print "\nTeams: ", teams
                print "Match Between Teams: ", match_between_teams
                print "Corners: ", corners
                print "Cards: ", cards
                print "BTTs :", btts
                print "goals :", goals
                print "Match status : ", mat_stat
                print get_player
                print "* "*10
            # break        

if __name__ == '__main__':
    # result = NLPExtractor().read_excel()
    result = NLPExtractor().read_pandas_excel()

Some Tweets and their results ::

@SkyBet Napoli & Psg to score 2+ goals, Napoli & Psg to have 6+ corners and both Lazio + Guingamp to have 20+ booking points #RequestABet

Teams:  [u'Lazio', u'PSG', u'Napoli']
Match Between Teams:  
Corners:  6+
Cards:  2+
BTTs : NA
goals : 2+
Match status :  
To carded :, To score : Psg, Napoli
* * * * * * * * * * 
@SkyBet Mertens to score, 2.5+goals, 4 corners each team & 20 + booking points each team #RequestABet

Teams:  []
Match Between Teams:  
Corners:  4
Cards:  None
BTTs : NA
goals : None
Match status :  
To carded :, To score : Mertens
* * * * * * * * * * 
@skybet here's my #REQUESTABET : BTTS, less than 12 corners, +20 booking points each, over 2.5 goals

Teams:  []
Match Between Teams:  
Corners:  12
Cards:  2+
BTTs : NA
goals : 3+
Match status :  
To carded :, To score : 
* * * * * * * * * * 
@SkyBet #RequestABet Mirallas to score and Everton to win, Over 2.5 goals, Holgate & Amartey to be carded, 7+ match corners

Teams:  [u'Everton']
Match Between Teams:  
Corners:  7+ match corners
Cards:  1+
BTTs : NA
goals : 2.5
Match status :  win
To carded :Amartey, Holgate, To score : Mirallas
* * * * * * * * * * 
@SkyBet Everton win, over 2.5 goals, 30+ booking points, 4+ corners each team #RequestABet

Teams:  [u'Everton']
Match Between Teams:  
Corners:  4+
Cards:  3+
BTTs : NA
goals : 3+
Match status :  win
To carded :, To score : 
* * * * * * * * * * 
@skybet here's my #REQUESTABET : more than 2.5 goals, under 12 corners, under 50 booking points, BTTS

Teams:  []
Match Between Teams:  
Corners:  12
Cards:  5
BTTs : NA
goals : 2.5
Match status :  
To carded :, To score : 
* * * * * * * * * * 
mikhitaryn to score oviedo and cattermole booked and a penslty awarded @SkyFootball #RequestABet
@SkyBet Man utd to score 2+ goals and win, Sunderland 20+ booking points, utd 6+ corners #RequestABet

Teams:  [u'Sunderland', 'utd']
Match Between Teams:  Sunderland vs utd
Corners:  6+
Cards:  2+
BTTs : NA
goals : 2+
Match status :  win
To carded :, To score : utd
* * * * * * * * * * 
@SkyBet #RequestABet Pogba to score outside the box, 20+ booking points each team, 11+ match corners, over 2.5 goals scored

Teams:  []
Match Between Teams:  
Corners:  11+ match corners
Cards:  2+
BTTs : NA
goals : 3+
Match status :  
To carded :, To score : Pogba
* * * * * * * * * * 
\$\endgroup\$
3
\$\begingroup\$

Here's some little nitpicks:

  • There really shouldn't be a space here: self.football_teams .append(name)

    Should be:

    self.football_teams.append(name)

  • Some of your class methods could be a staticmethod, for example remove_special_chars:

    @staticmethod
    def remove_special_chars(tokens):
        # Note the removal of 'self'
    

    As well as get_btts, check_over_under_condition, get_ngrams, etc.

  • Bare try: / except: statements (that don't specify an exception or exceptions) are generally a bad idea. They will also, for example, catch KeyboardInterrupts and for more complex applications, if another thread is being run simultaneously and it raises an error, it will be catched. If you know only specific exceptions will be raised, specify them.

  • Add docstrings to your code, so others understand how a function works (and what its ultimate goal is within a module / script). This may seem unecessary, but if you return to the code in a couple of months you will have probably forgotten how it worked, so docstrings really help.

  • Your get_btts() method is pretty weirdly defined:

    def get_btts(self, text):
        try:
            if 'no btts' in text:
                 btts = 'No'
            elif 'btts' in text:
                 btts = 'Yes'
                 return btts    # Why are you returning here?
            else:
                 btts = 'NA'
            return btts    # ~And here?
         except:    # There's little chance something would go wrong here..
            pass    # Well, that's not very helpful!
    
    @staticmethod
    def get_btts(text):
        """Check if the string 'btts' or 'no btts' occurs in `text`.
        Returns False if 'no btts' is found, True if 'btts' is found, else None.
        """
        if "no btts" in text:
            return False
        elif "btts" in text:
            return True
        return None
        # No need for an else: condition
    

    You can then check if get_btts:, elif get_btts == False:, else:

  • if-statements like if flag == 1: can be simplified to if flag, since Python automatically compares 'arguments' to True.

  • str.encode() automatically selects "utf-8" encoding if no arguments are given, so text.encode("utf-8") can be simplified to text.encode().

  • importing modules within functions is questionable (for me, since I always put my imports at the very top, no matter what), but it's up to you.

  • Two blank lines between imports and the first function or class and two blank lines in between each new function / class after that (not class methods).

  • importing should not be comma-seperated, so instead of:

    import re, math
    

    Do:

    import re
    import math
    
  • imports should be seperated into three categories (standard library imports, related third party imports, local imports) and each is seperated by a single space.

  • range() automatically assumes the starting number is 0, so there's no need to call range(0, upper). Also, I see you're using Python 2 (class name(object) use), in which case you should probably use xrange() for your ranges! Read this, though

\$\endgroup\$
  • 1
    \$\begingroup\$ Thanks, I really appreciate if you can review rest of the code as well. \$\endgroup\$ – user3449212 Aug 2 '17 at 16:43

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.