This is a follow up to my originaloriginal post:
Using /posts/ didn't work, so I changed it to the proper share link (adding your user ID so that you get the benefit if a bunch of people click the link)
""" file_formatter module
The class contained in this module validates a CSV file based on a set of internally
specified accepted formats and generates a JSON file containing normalized forms of the
valid lines from the CSV file.
Example:
The class in this module can be imported and passed an initial value for the input data
file from the command line like this:
$ python example_program.pypackage_name name_of_data_file.in
Classes:
FileFormatter: Takes an input file and output its valid lines to a result file.
"""
import json
import re
class FileFormatter:
""" Takes an input file and output its valid lines to a result file.
Validates the formatting of the lines from an input file and writes the set of valid lines
to an output file.
Attributes:
info_configs: A list containing lists of "accepted" configurations of the data from each line of the input file.
in_file_name: Name of the input file.
res_file_name: Name of the output file.
"""
info_configs = [["phone","color","zip"], ["color","zip","phone"], ["zip","phone","color"]]
def __init__(self,start_file_name,out_file_name):
"""Initialize FileFormatter class with the input and output file names."""
self.in_file_name = start_file_name
self.res_file_name = out_file_name
def validate_line(self, line):
"""Validates that each line is in the correct format.
Takes a line from a file, validate that the first two elements are properly formatted
names, then validates that the remaining elements (phone number, zip code, color)
in the line are properly formatted.
Args:
line: A line from a file
Returns:
A list of tokenized elements from the original line (string) in the correct order
according to the specified format. For example:
[Lastname, Firstname, (703)-742-0996, Blue, 10013] or
[Firstname, Lastname, Red, 11237, 703 955 0373] or
[Firstname, Lastname, 10013, 646 111 0101, Green]
If a value of None is returned, some element in the line wasn't in the correct format.
"""
line = tokenize(line)
if len(line) != 5:
return None
full_name = (line[0], line[1])
if not is_name(full_name):
return None
entry = { "color": "", "firstname": "", "lastname": "", "phonenumber": "", "zipcode": ""}
config = ["","",""]
phone_idx = 0
zip_idx = 0
color_idx = 0
for i in range(2,len(line)):
if is_phone_number(line[i]):
phone_idx = i-2
config[phone_idx] = "phone"
if is_zip_code(line[i]):
zip_idx = i-2
config[zip_idx] = "zip"
if is_color(line[i]):
color_idx = i-2
config[color_idx] = "color"
if config in self.info_configs: # if the phone number, zip code, and color have been found and are in correct order
if phone_idx == 0:
line[0], line[1] = line[1], line[0]
line = [token.strip(" ") for token in line]
line = [token.replace(",","") for token in line]
line[-1] = line[-1].replace("\n","")
entry["firstname"] = line[0]
entry["lastname"] = line[1]
entry["color"] = line[color_idx+2]
entry["phonenumber"] = line[phone_idx+2]
entry["zipcode"] = line[zip_idx+2]
return entry
return None
def parse_file(self):
"""Parses an input file, validates the formatting of its lines, and writes a JSON file with the properly formatted lines.
Iterates through the input file validating each line. Creates a dictionary that contains
a list of entries comprised of valid lines from the input file. Creates a JSON object
of normalized data sorted in ascending order by a tuple of (lastname, firstname) for each line.
"""
lines_dict = {}
json_dict = {}
errors = []
with open(self.in_file_name,'r') as info_file:
i = 0
for line_number, line in enumerate(info_file):
valid_line = self.validate_line(line)
if valid_line:
lines_dict[(valid_line["lastname"],valid_line["firstname"])] = valid_line
else:
errors.append(iline_number)
i += 1
json_dict["entries"] = [lines_dict[key] for key in sorted(lines_dict.keys(), reverse = True)] # sort by (lastname, firstname,) key value
json_dict["errors"] = errors
with open(self.res_file_name,'w') as out_file:
json.dump(json_dict, out_file, indent = 2)
# utility methods used by the FileFormatter class
def is_phone_number(token):
"""Determines if the passed in string represents a properly formatted 10-digit phone number.
Takes a string and validates that it matches one of two valid formats specified for a phone number.
Uses regular expression parsing to validate that the sequence of characters is an pattern match to one of
the valid formats.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted phone number.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return (re.match(r'\(\d{3}\)-\d{3}-\d{4}$', token) is not None or
re.match(r'\d{3} \d{3} \d{4}$', token) is not None)
def is_zip_code(token):
"""Determines if the passed in string represents a properly formatted 5-digit zip code.
Takes a string and using regular expression parsing to validate that it matches the valid
format specified for a zip code. Validates that the string doesn't contain more than 5 numbers.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted zip code.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return re.match(r'\d{5}$', token) is not None
def is_color(token):
"""Determines if the passed in string represents a color.
Takes a string and uses regular expression parsing to validate that
it matches the valid format specified for a color. Validates that
it is only a one word color.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted color.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return re.match(r'[a-z]*$', token) is not None
def is_name(name_tuple):
"""Determines if the first two elements in a file line (names) are correctly formatted.
Takes a tuple of elements and validates that they match one of two valid formats. Either both
words end in a comma or the second one does while the first one doesn't.
Args:
name_tuple: A tuple of two elements (first and last name) from a line in a file
Returns:
A boolean indicating if the elements (names) in the tuple are correctly formatted.
"""
names = (name_tuple[0].strip(" "), name_tuple[1].strip(" "))
name1_comma = False
name2_comma = False
for i in range(2):
curr_len = len(names[i])
for j in range(curr_len):
if not names[i][j].isalpha() and j < curr_len -1:
return False
if j == curr_len - 1 and i == 0 and names[i][j] == ',':
name1_comma = True
if j == curr_len - 1 and i == 1 and names[i][j] == ',':
name2_comma = True
comma_first_case = name1_comma and name2_comma # both have commas
comma_second_case = not name1_comma and name2_comma # name2 has comma, name 1 doesnt
if not (comma_first_case or comma_second_case):
return False
return True
def tokenize(line):
"""Splits the passed in string on the delimiter and return a list of tokens.
Takes a string and splits it on a delimter while maintaining the delimiter in its
original position in the string. If the first word in the string doesn't end with a comma,
the split operation will yield four tokens instead of five so the first two words (names) are broken
up by the space character.
Args:
line: A string to be broken up into tokens based on a delimiter.
Returns:
A list of tokens (words) from the passed in line.
"""
delim = ","
tokens = [e + delim for e in line.split(delim) if e]
if len(tokens) == 4:
names = tokens[0].split(" ")
names[0] += delim
names[1] = " " + names[1]
info = tokens[1:]
tokens = []
tokens.extend(names)
tokens.extend(info)
return tokens
""" file_formatter module
The class contained in this module validates a CSV file based on a set of internally
specified accepted formats and generates a JSON file containing normalized forms of the
valid lines from the CSV file.
Example:
The class in this module can be imported and passed an initial value for the input data
file from the command line like this:
$ python example_program.py name_of_data_file.in
Classes:
FileFormatter: Takes an input file and output its valid lines to a result file.
"""
import json
import re
class FileFormatter:
""" Takes an input file and output its valid lines to a result file.
Validates the formatting of the lines from an input file and writes the set of valid lines
to an output file.
Attributes:
info_configs: A list containing lists of "accepted" configurations of the data from each line of the input file.
in_file_name: Name of the input file.
res_file_name: Name of the output file.
"""
info_configs = [["phone","color","zip"], ["color","zip","phone"], ["zip","phone","color"]]
def __init__(self,start_file_name,out_file_name):
"""Initialize FileFormatter class with the input and output file names."""
self.in_file_name = start_file_name
self.res_file_name = out_file_name
def validate_line(self, line):
"""Validates that each line is in the correct format.
Takes a line from a file, validate that the first two elements are properly formatted
names, then validates that the remaining elements (phone number, zip code, color)
in the line are properly formatted.
Args:
line: A line from a file
Returns:
A list of tokenized elements from the original line (string) in the correct order
according to the specified format. For example:
[Lastname, Firstname, (703)-742-0996, Blue, 10013] or
[Firstname, Lastname, Red, 11237, 703 955 0373] or
[Firstname, Lastname, 10013, 646 111 0101, Green]
If a value of None is returned, some element in the line wasn't in the correct format.
"""
line = tokenize(line)
if len(line) != 5:
return None
full_name = (line[0], line[1])
if not is_name(full_name):
return None
entry = { "color": "", "firstname": "", "lastname": "", "phonenumber": "", "zipcode": ""}
config = ["","",""]
phone_idx = 0
zip_idx = 0
color_idx = 0
for i in range(2,len(line)):
if is_phone_number(line[i]):
phone_idx = i-2
config[phone_idx] = "phone"
if is_zip_code(line[i]):
zip_idx = i-2
config[zip_idx] = "zip"
if is_color(line[i]):
color_idx = i-2
config[color_idx] = "color"
if config in self.info_configs: # if the phone number, zip code, and color have been found and are in correct order
if phone_idx == 0:
line[0], line[1] = line[1], line[0]
line = [token.strip(" ") for token in line]
line = [token.replace(",","") for token in line]
line[-1] = line[-1].replace("\n","")
entry["firstname"] = line[0]
entry["lastname"] = line[1]
entry["color"] = line[color_idx+2]
entry["phonenumber"] = line[phone_idx+2]
entry["zipcode"] = line[zip_idx+2]
return entry
return None
def parse_file(self):
"""Parses an input file, validates the formatting of its lines, and writes a JSON file with the properly formatted lines.
Iterates through the input file validating each line. Creates a dictionary that contains
a list of entries comprised of valid lines from the input file. Creates a JSON object
of normalized data sorted in ascending order by a tuple of (lastname, firstname) for each line.
"""
lines_dict = {}
json_dict = {}
errors = []
with open(self.in_file_name,'r') as info_file:
i = 0
for line in info_file:
valid_line = self.validate_line(line)
if valid_line:
lines_dict[(valid_line["lastname"],valid_line["firstname"])] = valid_line
else:
errors.append(i)
i += 1
json_dict["entries"] = [lines_dict[key] for key in sorted(lines_dict.keys(), reverse = True)] # sort by (lastname, firstname,) key value
json_dict["errors"] = errors
with open(self.res_file_name,'w') as out_file:
json.dump(json_dict, out_file, indent = 2)
# utility methods used by the FileFormatter class
def is_phone_number(token):
"""Determines if the passed in string represents a properly formatted 10-digit phone number.
Takes a string and validates that it matches one of two valid formats specified for a phone number.
Uses regular expression parsing to validate that the sequence of characters is an pattern match to one of
the valid formats.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted phone number.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return (re.match(r'\(\d{3}\)-\d{3}-\d{4}$', token) is not None or
re.match(r'\d{3} \d{3} \d{4}$', token) is not None)
def is_color(token):
"""Determines if the passed in string represents a color.
Takes a string and uses regular expression parsing to validate that
it matches the valid format specified for a color. Validates that
it is only a one word color.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted color.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return re.match(r'[a-z]*$', token) is not None
def is_name(name_tuple):
"""Determines if the first two elements in a file line (names) are correctly formatted.
Takes a tuple of elements and validates that they match one of two valid formats. Either both
words end in a comma or the second one does while the first one doesn't.
Args:
name_tuple: A tuple of two elements (first and last name) from a line in a file
Returns:
A boolean indicating if the elements (names) in the tuple are correctly formatted.
"""
names = (name_tuple[0].strip(" "), name_tuple[1].strip(" "))
name1_comma = False
name2_comma = False
for i in range(2):
curr_len = len(names[i])
for j in range(curr_len):
if not names[i][j].isalpha() and j < curr_len -1:
return False
if j == curr_len - 1 and i == 0 and names[i][j] == ',':
name1_comma = True
if j == curr_len - 1 and i == 1 and names[i][j] == ',':
name2_comma = True
comma_first_case = name1_comma and name2_comma # both have commas
comma_second_case = not name1_comma and name2_comma # name2 has comma, name 1 doesnt
if not (comma_first_case or comma_second_case):
return False
return True
def tokenize(line):
"""Splits the passed in string on the delimiter and return a list of tokens.
Takes a string and splits it on a delimter while maintaining the delimiter in its
original position in the string. If the first word in the string doesn't end with a comma,
the split operation will yield four tokens instead of five so the first two words (names) are broken
up by the space character.
Args:
line: A string to be broken up into tokens based on a delimiter.
Returns:
A list of tokens (words) from the passed in line.
"""
delim = ","
tokens = [e + delim for e in line.split(delim) if e]
if len(tokens) == 4:
names = tokens[0].split(" ")
names[0] += delim
names[1] = " " + names[1]
info = tokens[1:]
tokens = []
tokens.extend(names)
tokens.extend(info)
return tokens
""" file_formatter module
The class contained in this module validates a CSV file based on a set of internally
specified accepted formats and generates a JSON file containing normalized forms of the
valid lines from the CSV file.
Example:
The class in this module can be imported and passed an initial value for the input data
file from the command line like this:
$ python package_name name_of_data_file.in
Classes:
FileFormatter: Takes an input file and output its valid lines to a result file.
"""
import json
import re
class FileFormatter:
""" Takes an input file and output its valid lines to a result file.
Validates the formatting of the lines from an input file and writes the set of valid lines
to an output file.
Attributes:
info_configs: A list containing lists of "accepted" configurations of the data from each line of the input file.
in_file_name: Name of the input file.
res_file_name: Name of the output file.
"""
info_configs = [["phone","color","zip"], ["color","zip","phone"], ["zip","phone","color"]]
def __init__(self,start_file_name,out_file_name):
"""Initialize FileFormatter class with the input and output file names."""
self.in_file_name = start_file_name
self.res_file_name = out_file_name
def validate_line(self, line):
"""Validates that each line is in the correct format.
Takes a line from a file, validate that the first two elements are properly formatted
names, then validates that the remaining elements (phone number, zip code, color)
in the line are properly formatted.
Args:
line: A line from a file
Returns:
A list of tokenized elements from the original line (string) in the correct order
according to the specified format. For example:
[Lastname, Firstname, (703)-742-0996, Blue, 10013] or
[Firstname, Lastname, Red, 11237, 703 955 0373] or
[Firstname, Lastname, 10013, 646 111 0101, Green]
If a value of None is returned, some element in the line wasn't in the correct format.
"""
line = tokenize(line)
if len(line) != 5:
return None
full_name = (line[0], line[1])
if not is_name(full_name):
return None
entry = { "color": "", "firstname": "", "lastname": "", "phonenumber": "", "zipcode": ""}
config = ["","",""]
phone_idx = 0
zip_idx = 0
color_idx = 0
for i in range(2,len(line)):
if is_phone_number(line[i]):
phone_idx = i-2
config[phone_idx] = "phone"
if is_zip_code(line[i]):
zip_idx = i-2
config[zip_idx] = "zip"
if is_color(line[i]):
color_idx = i-2
config[color_idx] = "color"
if config in self.info_configs: # if the phone number, zip code, and color have been found and are in correct order
if phone_idx == 0:
line[0], line[1] = line[1], line[0]
line = [token.strip(" ") for token in line]
line = [token.replace(",","") for token in line]
line[-1] = line[-1].replace("\n","")
entry["firstname"] = line[0]
entry["lastname"] = line[1]
entry["color"] = line[color_idx+2]
entry["phonenumber"] = line[phone_idx+2]
entry["zipcode"] = line[zip_idx+2]
return entry
return None
def parse_file(self):
"""Parses an input file, validates the formatting of its lines, and writes a JSON file with the properly formatted lines.
Iterates through the input file validating each line. Creates a dictionary that contains
a list of entries comprised of valid lines from the input file. Creates a JSON object
of normalized data sorted in ascending order by a tuple of (lastname, firstname) for each line.
"""
lines_dict = {}
json_dict = {}
errors = []
with open(self.in_file_name,'r') as info_file:
for line_number, line in enumerate(info_file):
valid_line = self.validate_line(line)
if valid_line:
lines_dict[(valid_line["lastname"],valid_line["firstname"])] = valid_line
else:
errors.append(line_number)
json_dict["entries"] = [lines_dict[key] for key in sorted(lines_dict.keys(), reverse = True)] # sort by (lastname, firstname,) key value
json_dict["errors"] = errors
with open(self.res_file_name,'w') as out_file:
json.dump(json_dict, out_file, indent = 2)
# utility methods used by the FileFormatter class
def is_phone_number(token):
"""Determines if the passed in string represents a properly formatted 10-digit phone number.
Takes a string and validates that it matches one of two valid formats specified for a phone number.
Uses regular expression parsing to validate that the sequence of characters is an pattern match to one of
the valid formats.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted phone number.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return (re.match(r'\(\d{3}\)-\d{3}-\d{4}$', token) is not None or
re.match(r'\d{3} \d{3} \d{4}$', token) is not None)
def is_zip_code(token):
"""Determines if the passed in string represents a properly formatted 5-digit zip code.
Takes a string and using regular expression parsing to validate that it matches the valid
format specified for a zip code. Validates that the string doesn't contain more than 5 numbers.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted zip code.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return re.match(r'\d{5}$', token) is not None
def is_color(token):
"""Determines if the passed in string represents a color.
Takes a string and uses regular expression parsing to validate that
it matches the valid format specified for a color. Validates that
it is only a one word color.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted color.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return re.match(r'[a-z]*$', token) is not None
def is_name(name_tuple):
"""Determines if the first two elements in a file line (names) are correctly formatted.
Takes a tuple of elements and validates that they match one of two valid formats. Either both
words end in a comma or the second one does while the first one doesn't.
Args:
name_tuple: A tuple of two elements (first and last name) from a line in a file
Returns:
A boolean indicating if the elements (names) in the tuple are correctly formatted.
"""
names = (name_tuple[0].strip(" "), name_tuple[1].strip(" "))
name1_comma = False
name2_comma = False
for i in range(2):
curr_len = len(names[i])
for j in range(curr_len):
if not names[i][j].isalpha() and j < curr_len -1:
return False
if j == curr_len - 1 and i == 0 and names[i][j] == ',':
name1_comma = True
if j == curr_len - 1 and i == 1 and names[i][j] == ',':
name2_comma = True
comma_first_case = name1_comma and name2_comma # both have commas
comma_second_case = not name1_comma and name2_comma # name2 has comma, name 1 doesnt
if not (comma_first_case or comma_second_case):
return False
return True
def tokenize(line):
"""Splits the passed in string on the delimiter and return a list of tokens.
Takes a string and splits it on a delimter while maintaining the delimiter in its
original position in the string. If the first word in the string doesn't end with a comma,
the split operation will yield four tokens instead of five so the first two words (names) are broken
up by the space character.
Args:
line: A string to be broken up into tokens based on a delimiter.
Returns:
A list of tokens (words) from the passed in line.
"""
delim = ","
tokens = [e + delim for e in line.split(delim) if e]
if len(tokens) == 4:
names = tokens[0].split(" ")
names[0] += delim
names[1] = " " + names[1]
info = tokens[1:]
tokens = []
tokens.extend(names)
tokens.extend(info)
return tokens
""" file_formatter module
The class contained in this module validates a CSV file based on a set of internally
specified accepted formats and generates a JSON file containing normalized forms of the
valid lines from the CSV file.
Example:
The class in this module can be imported and passed an initial value for the input data
file from the command line like this:
$ python example_program.py name_of_data_file.in
Classes:
FileFormatter: Takes an input file and output its valid lines to a result file.
"""
import json
import re
class FileFormatter:
""" Takes an input file and output its valid lines to a result file.
Validates the formatting of the lines from an input file and writes the set of valid lines
to an output file.
Attributes:
info_configs: A list containing lists of "accepted" configurations of the data from each line of the input file.
in_file_name: Name of the input file.
res_file_name: Name of the output file.
"""
info_configs = [["phone","color","zip"], ["color","zip","phone"], ["zip","phone","color"]]
def __init__(self,start_file_name,out_file_name):
"""Initialize FileFormatter class with the input and output file names."""
self.in_file_name = start_file_name
self.res_file_name = out_file_name
def validate_line(self, line):
"""Validates that each line is in the correct format.
Takes a line from a file, validate that the first two elements are properly formatted
names, then validates that the remaining elements (phone number, zip code, color)
in the line are properly formatted.
Args:
line: A line from a file
Returns:
A list of tokenized elements from the original line (string) in the correct order
according to the specified format. For example:
[Lastname, Firstname, (703)-742-0996, Blue, 10013] or
[Firstname, Lastname, Red, 11237, 703 955 0373] or
[Firstname, Lastname, 10013, 646 111 0101, Green]
If a value of None is returned, some element in the line wasn't in the correct format.
"""
line = tokenize(line)
if len(line) != 5:
return None
full_name = (line[0], line[1])
if not is_name(full_name):
return None
entry = { "color": "", "firstname": "", "lastname": "", "phonenumber": "", "zipcode": ""}
config = ["","",""]
phone_idx = 0
zip_idx = 0
color_idx = 0
for i in range(2,len(line)):
if is_phone_number(line[i]):
phone_idx = i-2
config[phone_idx] = "phone"
if is_zip_code(line[i]):
zip_idx = i-2
config[zip_idx] = "zip"
if is_color(line[i]):
color_idx = i-2
config[color_idx] = "color"
if config in self.info_configs: # if the phone number, zip code, and color have been found and are in correct order
if phone_idx == 0:
line[0], line[1] = line[1], line[0]
line = [token.strip(" ") for token in line]
line = [token.replace(",","") for token in line]
line[len(line)line[-1] = line[len(line)line[-1].replace("\n","")
entry["firstname"] = line[0]
entry["lastname"] = line[1]
entry["color"] = line[color_idx+2]
entry["phonenumber"] = line[phone_idx+2]
entry["zipcode"] = line[zip_idx+2]
return entry
return None
def parse_file(self):
"""Parses an input file, validates the formatting of its lines, and writes a JSON file with the properly formatted lines.
Iterates through the input file validating each line. Creates a dictionary that contains
a list of entries comprised of valid lines from the input file. Creates a JSON object
of normalized data sorted in ascending order by a tuple of (lastname, firstname) for each line.
"""
lines_dict = {}
json_dict = {}
errors = []
with open(self.in_file_name,'r') as info_file:
i = 0
for line in info_file:
valid_line = self.validate_line(line)
if valid_line:
lines_dict[(valid_line["lastname"],valid_line["firstname"])] = valid_line
else:
errors.append(i)
i += 1
json_dict["entries"] = [lines_dict[key] for key in sorted(lines_dict.keys(), reverse = True)] # sort by (lastname, firstname,) key value
json_dict["errors"] = errors
with open(self.res_file_name,'w') as out_file:
json.dump(json_dict, out_file, indent = 2)
# utility methods used by the FileFormatter class
def is_phone_number(token):
"""Determines if the passed in string represents a properly formatted 10-digit phone number.
Takes a string and validates that it matches one of two valid formats specified for a phone number.
Uses regular expression parsing to validate that the sequence of characters is an pattern match to one of
the valid formats.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted phone number.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return (re.match(r'\(\d{3}\)-\d{3}-\d{4}$', token) is not None or
re.match(r'\d{3} \d{3} \d{4}$', token) is not None)
def is_color(token):
"""Determines if the passed in string represents a color.
Takes a string and uses regular expression parsing to validate that
it matches the valid format specified for a color. Validates that
it is only a one word color.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted color.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return re.match(r'[a-z]*$', token) is not None
def is_name(name_tuple):
"""Determines if the first two elements in a file line (names) are correctly formatted.
Takes a tuple of elements and validates that they match one of two valid formats. Either both
words end in a comma or the second one does while the first one doesn't.
Args:
name_tuple: A tuple of two elements (first and last name) from a line in a file
Returns:
A boolean indicating if the elements (names) in the tuple are correctly formatted.
"""
names = (name_tuple[0].strip(" "), name_tuple[1].strip(" "))
comma_first_case = False
comma_second_case = False
name1_comma = False
name2_comma = False
for i in range(2):
curr_len = len(names[i])
for j in range(curr_len):
if not names[i][j].isalpha() and j < curr_len -1:
return False
if j == curr_len - 1 and i == 0 and names[i][j] == ',':
name1_comma = True
if j == curr_len - 1 and i == 1 and names[i][j] == ',':
name2_comma = True
comma_first_case = name1_comma and name2_comma # both have commas
comma_second_case = not name1_comma and name2_comma # name2 has comma, name 1 doesnt
if not (comma_first_case or comma_second_case):
return False
return True
def tokenize(line):
"""Splits the passed in string on the delimiter and return a list of tokens.
Takes a string and splits it on a delimter while maintaining the delimiter in its
original position in the string. If the first word in the string doesn't end with a comma,
the split operation will yield four tokens instead of five so the first two words (names) are broken
up by the space character.
Args:
line: A string to be broken up into tokens based on a delimiter.
Returns:
A list of tokens (words) from the passed in line.
"""
delim = ","
tokens = [e + delim for e in line.split(delim) if e]
if len(tokens) == 4:
names = tokens[0].split(" ")
names[0] = names[0] ++= delim
names[1] = " " + names[1]
info = tokens[1:]
tokens = []
tokens.extend(names)
tokens.extend(info)
return tokens
""" file_formatter module
The class contained in this module validates a CSV file based on a set of internally
specified accepted formats and generates a JSON file containing normalized forms of the
valid lines from the CSV file.
Example:
The class in this module can be imported and passed an initial value for the input data
file from the command line like this:
$ python example_program.py name_of_data_file.in
Classes:
FileFormatter: Takes an input file and output its valid lines to a result file.
"""
import json
import re
class FileFormatter:
""" Takes an input file and output its valid lines to a result file.
Validates the formatting of the lines from an input file and writes the set of valid lines
to an output file.
Attributes:
info_configs: A list containing lists of "accepted" configurations of the data from each line of the input file.
in_file_name: Name of the input file.
res_file_name: Name of the output file.
"""
info_configs = [["phone","color","zip"], ["color","zip","phone"], ["zip","phone","color"]]
def __init__(self,start_file_name,out_file_name):
"""Initialize FileFormatter class with the input and output file names."""
self.in_file_name = start_file_name
self.res_file_name = out_file_name
def validate_line(self, line):
"""Validates that each line is in the correct format.
Takes a line from a file, validate that the first two elements are properly formatted
names, then validates that the remaining elements (phone number, zip code, color)
in the line are properly formatted.
Args:
line: A line from a file
Returns:
A list of tokenized elements from the original line (string) in the correct order
according to the specified format. For example:
[Lastname, Firstname, (703)-742-0996, Blue, 10013] or
[Firstname, Lastname, Red, 11237, 703 955 0373] or
[Firstname, Lastname, 10013, 646 111 0101, Green]
If a value of None is returned, some element in the line wasn't in the correct format.
"""
line = tokenize(line)
if len(line) != 5:
return None
full_name = (line[0], line[1])
if not is_name(full_name):
return None
entry = { "color": "", "firstname": "", "lastname": "", "phonenumber": "", "zipcode": ""}
config = ["","",""]
phone_idx = 0
zip_idx = 0
color_idx = 0
for i in range(2,len(line)):
if is_phone_number(line[i]):
phone_idx = i-2
config[phone_idx] = "phone"
if is_zip_code(line[i]):
zip_idx = i-2
config[zip_idx] = "zip"
if is_color(line[i]):
color_idx = i-2
config[color_idx] = "color"
if config in self.info_configs: # if the phone number, zip code, and color have been found and are in correct order
if phone_idx == 0:
line[0], line[1] = line[1], line[0]
line = [token.strip(" ") for token in line]
line = [token.replace(",","") for token in line]
line[len(line)-1] = line[len(line)-1].replace("\n","")
entry["firstname"] = line[0]
entry["lastname"] = line[1]
entry["color"] = line[color_idx+2]
entry["phonenumber"] = line[phone_idx+2]
entry["zipcode"] = line[zip_idx+2]
return entry
return None
def parse_file(self):
"""Parses an input file, validates the formatting of its lines, and writes a JSON file with the properly formatted lines.
Iterates through the input file validating each line. Creates a dictionary that contains
a list of entries comprised of valid lines from the input file. Creates a JSON object
of normalized data sorted in ascending order by a tuple of (lastname, firstname) for each line.
"""
lines_dict = {}
json_dict = {}
errors = []
with open(self.in_file_name,'r') as info_file:
i = 0
for line in info_file:
valid_line = self.validate_line(line)
if valid_line:
lines_dict[(valid_line["lastname"],valid_line["firstname"])] = valid_line
else:
errors.append(i)
i += 1
json_dict["entries"] = [lines_dict[key] for key in sorted(lines_dict.keys(), reverse = True)] # sort by (lastname, firstname,) key value
json_dict["errors"] = errors
with open(self.res_file_name,'w') as out_file:
json.dump(json_dict, out_file, indent = 2)
# utility methods used by the FileFormatter class
def is_phone_number(token):
"""Determines if the passed in string represents a properly formatted 10-digit phone number.
Takes a string and validates that it matches one of two valid formats specified for a phone number.
Uses regular expression parsing to validate that the sequence of characters is an pattern match to one of
the valid formats.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted phone number.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return (re.match(r'\(\d{3}\)-\d{3}-\d{4}$', token) is not None or
re.match(r'\d{3} \d{3} \d{4}$', token) is not None)
def is_color(token):
"""Determines if the passed in string represents a color.
Takes a string and uses regular expression parsing to validate that
it matches the valid format specified for a color. Validates that
it is only a one word color.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted color.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return re.match(r'[a-z]*$', token) is not None
def is_name(name_tuple):
"""Determines if the first two elements in a file line (names) are correctly formatted.
Takes a tuple of elements and validates that they match one of two valid formats. Either both
words end in a comma or the second one does while the first one doesn't.
Args:
name_tuple: A tuple of two elements (first and last name) from a line in a file
Returns:
A boolean indicating if the elements (names) in the tuple are correctly formatted.
"""
names = (name_tuple[0].strip(" "), name_tuple[1].strip(" "))
comma_first_case = False
comma_second_case = False
name1_comma = False
name2_comma = False
for i in range(2):
curr_len = len(names[i])
for j in range(curr_len):
if not names[i][j].isalpha() and j < curr_len -1:
return False
if j == curr_len - 1 and i == 0 and names[i][j] == ',':
name1_comma = True
if j == curr_len - 1 and i == 1 and names[i][j] == ',':
name2_comma = True
comma_first_case = name1_comma and name2_comma # both have commas
comma_second_case = not name1_comma and name2_comma # name2 has comma, name 1 doesnt
if not (comma_first_case or comma_second_case):
return False
return True
def tokenize(line):
"""Splits the passed in string on the delimiter and return a list of tokens.
Takes a string and splits it on a delimter while maintaining the delimiter in its
original position in the string. If the first word in the string doesn't end with a comma,
the split operation will yield four tokens instead of five so the first two words (names) are broken
up by the space character.
Args:
line: A string to be broken up into tokens based on a delimiter.
Returns:
A list of tokens (words) from the passed in line.
"""
delim = ","
tokens = [e + delim for e in line.split(delim) if e]
if len(tokens) == 4:
names = tokens[0].split(" ")
names[0] = names[0] + delim
names[1] = " " + names[1]
info = tokens[1:]
tokens = []
tokens.extend(names)
tokens.extend(info)
return tokens
""" file_formatter module
The class contained in this module validates a CSV file based on a set of internally
specified accepted formats and generates a JSON file containing normalized forms of the
valid lines from the CSV file.
Example:
The class in this module can be imported and passed an initial value for the input data
file from the command line like this:
$ python example_program.py name_of_data_file.in
Classes:
FileFormatter: Takes an input file and output its valid lines to a result file.
"""
import json
import re
class FileFormatter:
""" Takes an input file and output its valid lines to a result file.
Validates the formatting of the lines from an input file and writes the set of valid lines
to an output file.
Attributes:
info_configs: A list containing lists of "accepted" configurations of the data from each line of the input file.
in_file_name: Name of the input file.
res_file_name: Name of the output file.
"""
info_configs = [["phone","color","zip"], ["color","zip","phone"], ["zip","phone","color"]]
def __init__(self,start_file_name,out_file_name):
"""Initialize FileFormatter class with the input and output file names."""
self.in_file_name = start_file_name
self.res_file_name = out_file_name
def validate_line(self, line):
"""Validates that each line is in the correct format.
Takes a line from a file, validate that the first two elements are properly formatted
names, then validates that the remaining elements (phone number, zip code, color)
in the line are properly formatted.
Args:
line: A line from a file
Returns:
A list of tokenized elements from the original line (string) in the correct order
according to the specified format. For example:
[Lastname, Firstname, (703)-742-0996, Blue, 10013] or
[Firstname, Lastname, Red, 11237, 703 955 0373] or
[Firstname, Lastname, 10013, 646 111 0101, Green]
If a value of None is returned, some element in the line wasn't in the correct format.
"""
line = tokenize(line)
if len(line) != 5:
return None
full_name = (line[0], line[1])
if not is_name(full_name):
return None
entry = { "color": "", "firstname": "", "lastname": "", "phonenumber": "", "zipcode": ""}
config = ["","",""]
phone_idx = 0
zip_idx = 0
color_idx = 0
for i in range(2,len(line)):
if is_phone_number(line[i]):
phone_idx = i-2
config[phone_idx] = "phone"
if is_zip_code(line[i]):
zip_idx = i-2
config[zip_idx] = "zip"
if is_color(line[i]):
color_idx = i-2
config[color_idx] = "color"
if config in self.info_configs: # if the phone number, zip code, and color have been found and are in correct order
if phone_idx == 0:
line[0], line[1] = line[1], line[0]
line = [token.strip(" ") for token in line]
line = [token.replace(",","") for token in line]
line[-1] = line[-1].replace("\n","")
entry["firstname"] = line[0]
entry["lastname"] = line[1]
entry["color"] = line[color_idx+2]
entry["phonenumber"] = line[phone_idx+2]
entry["zipcode"] = line[zip_idx+2]
return entry
return None
def parse_file(self):
"""Parses an input file, validates the formatting of its lines, and writes a JSON file with the properly formatted lines.
Iterates through the input file validating each line. Creates a dictionary that contains
a list of entries comprised of valid lines from the input file. Creates a JSON object
of normalized data sorted in ascending order by a tuple of (lastname, firstname) for each line.
"""
lines_dict = {}
json_dict = {}
errors = []
with open(self.in_file_name,'r') as info_file:
i = 0
for line in info_file:
valid_line = self.validate_line(line)
if valid_line:
lines_dict[(valid_line["lastname"],valid_line["firstname"])] = valid_line
else:
errors.append(i)
i += 1
json_dict["entries"] = [lines_dict[key] for key in sorted(lines_dict.keys(), reverse = True)] # sort by (lastname, firstname,) key value
json_dict["errors"] = errors
with open(self.res_file_name,'w') as out_file:
json.dump(json_dict, out_file, indent = 2)
# utility methods used by the FileFormatter class
def is_phone_number(token):
"""Determines if the passed in string represents a properly formatted 10-digit phone number.
Takes a string and validates that it matches one of two valid formats specified for a phone number.
Uses regular expression parsing to validate that the sequence of characters is an pattern match to one of
the valid formats.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted phone number.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return (re.match(r'\(\d{3}\)-\d{3}-\d{4}$', token) is not None or
re.match(r'\d{3} \d{3} \d{4}$', token) is not None)
def is_color(token):
"""Determines if the passed in string represents a color.
Takes a string and uses regular expression parsing to validate that
it matches the valid format specified for a color. Validates that
it is only a one word color.
Args:
token: A fragment of a line of a file
Returns:
A boolean indicating if the string is a properly formatted color.
"""
token = token.strip(" ")
token = token.replace("\n","")
token = token.replace(",","")
return re.match(r'[a-z]*$', token) is not None
def is_name(name_tuple):
"""Determines if the first two elements in a file line (names) are correctly formatted.
Takes a tuple of elements and validates that they match one of two valid formats. Either both
words end in a comma or the second one does while the first one doesn't.
Args:
name_tuple: A tuple of two elements (first and last name) from a line in a file
Returns:
A boolean indicating if the elements (names) in the tuple are correctly formatted.
"""
names = (name_tuple[0].strip(" "), name_tuple[1].strip(" "))
name1_comma = False
name2_comma = False
for i in range(2):
curr_len = len(names[i])
for j in range(curr_len):
if not names[i][j].isalpha() and j < curr_len -1:
return False
if j == curr_len - 1 and i == 0 and names[i][j] == ',':
name1_comma = True
if j == curr_len - 1 and i == 1 and names[i][j] == ',':
name2_comma = True
comma_first_case = name1_comma and name2_comma # both have commas
comma_second_case = not name1_comma and name2_comma # name2 has comma, name 1 doesnt
if not (comma_first_case or comma_second_case):
return False
return True
def tokenize(line):
"""Splits the passed in string on the delimiter and return a list of tokens.
Takes a string and splits it on a delimter while maintaining the delimiter in its
original position in the string. If the first word in the string doesn't end with a comma,
the split operation will yield four tokens instead of five so the first two words (names) are broken
up by the space character.
Args:
line: A string to be broken up into tokens based on a delimiter.
Returns:
A list of tokens (words) from the passed in line.
"""
delim = ","
tokens = [e + delim for e in line.split(delim) if e]
if len(tokens) == 4:
names = tokens[0].split(" ")
names[0] += delim
names[1] = " " + names[1]
info = tokens[1:]
tokens = []
tokens.extend(names)
tokens.extend(info)
return tokens
Loading
lang-py