Parse complex text files using Python

Question

I'm looking for a simple way of parsing complex text files into a pandas DataFrame. Below is a sample file, what I want the result to look like after parsing, and my current method.

Is there any way to make it more concise/faster/more pythonic/more readable?

I have also put this question on Stack Overflow. I eventually wrote a blog article to explain this to beginners.

Here is a sample file:

Sample text

A selection of students from Riverdale High and Hogwarts took part in a quiz. This is a record of their scores.

School = Riverdale High
Grade = 1
Student number, Name
0, Phoebe
1, Rachel

Student number, Score
0, 3
1, 7

Grade = 2
Student number, Name
0, Angela
1, Tristan
2, Aurora

Student number, Score
0, 6
1, 3
2, 9

School = Hogwarts
Grade = 1
Student number, Name
0, Ginny
1, Luna

Student number, Score
0, 8
1, 7

Grade = 2
Student number, Name
0, Harry
1, Hermione

Student number, Score
0, 5
1, 10

Grade = 3
Student number, Name
0, Fred
1, George

Student number, Score
0, 0
1, 0

Here is what I want the result to look like after parsing:

                                         Name  Score
School         Grade Student number                 
Hogwarts       1     0                  Ginny      8
                     1                   Luna      7
               2     0                  Harry      5
                     1               Hermione     10
               3     0                   Fred      0
                     1                 George      0
Riverdale High 1     0                 Phoebe      3
                     1                 Rachel      7
               2     0                 Angela      6
                     1                Tristan      3
                     2                 Aurora      9

Here is how I currently parse it:

import re
import pandas as pd


def parse(filepath):
    """
    Parse text at given filepath

    Parameters
    ----------
    filepath : str
        Filepath for file to be parsed

    Returns
    -------
    data : pd.DataFrame
        Parsed data

    """

    data = []
    with open(filepath, 'r') as file:
        line = file.readline()
        while line:
            reg_match = _RegExLib(line)

            if reg_match.school:
                school = reg_match.school.group(1)

            if reg_match.grade:
                grade = reg_match.grade.group(1)
                grade = int(grade)

            if reg_match.name_score:
                value_type = reg_match.name_score.group(1)
                line = file.readline()
                while line.strip():
                    number, value = line.strip().split(',')
                    value = value.strip()
                    dict_of_data = {
                        'School': school,
                        'Grade': grade,
                        'Student number': number,
                        value_type: value
                    }
                    data.append(dict_of_data)
                    line = file.readline()

            line = file.readline()

        data = pd.DataFrame(data)
        data.set_index(['School', 'Grade', 'Student number'], inplace=True)
        # consolidate df to remove nans
        data = data.groupby(level=data.index.names).first()
        # upgrade Score from float to integer
        data = data.apply(pd.to_numeric, errors='ignore')
    return data


class _RegExLib:
    """Set up regular expressions"""
    # use https://regexper.com to visualise these if required
    _reg_school = re.compile('School = (.*)\n')
    _reg_grade = re.compile('Grade = (.*)\n')
    _reg_name_score = re.compile('(Name|Score)')

    def __init__(self, line):
        # check whether line has a positive match with all of the regular expressions
        self.school = self._reg_school.match(line)
        self.grade = self._reg_grade.match(line)
        self.name_score = self._reg_name_score.search(line)


if __name__ == '__main__':
    filepath = 'sample.txt'
    data = parse(filepath)
    print(data)

Answer 1

There are a few performance tricks we can apply here:

• add __slots__ to the class definition should help with memory and performance as well:

  class _RegExLib:
      """Set up regular expressions"""
      # use https://regexper.com to visualise these if required
      _reg_school = re.compile(r'School = (.*)\n')
      _reg_grade = re.compile(r'Grade = (.*)\n')
      _reg_name_score = re.compile(r'(Name|Score)')
  
      __slots__ = ['school', 'grade', 'name_score']
  
      def __init__(self, line):
          # check whether line has a positive match with all of the regular expressions
          self.school = self._reg_school.match(line)
          self.grade = self._reg_grade.match(line)
          self.name_score = self._reg_name_score.search(line)
  

• using next() instead of .readline() should be faster as it uses a lookahead buffer internally:

  with open(filepath, 'r') as file:
      line = next(file)
      while line:
          reg_match = _RegExLib(line)
  
          if reg_match.school:
              school = reg_match.school.group(1)
  
          if reg_match.grade:
              grade = reg_match.grade.group(1)
              grade = int(grade)
  
          if reg_match.name_score:
              value_type = reg_match.name_score.group(1)
              line = next(file, None)
              while line and line.strip():
                  number, value = line.strip().split(',')
                  value = value.strip()
                  dict_of_data = {
                      'School': school,
                      'Grade': grade,
                      'Student number': number,
                      value_type: value
                  }
                  data.append(dict_of_data)
                  line = next(file, None)
  
          line = next(file, None)
  

---

Some of the code style and other notes:

• file is a builtin (in Python 2.x only), consider a different variable name
• define your regular expression strings as raw strings
• you can probably replace .* wildcard with a more concrete \d+ for the "grade" regex: Grade = (\d+)\n