2
\$\begingroup\$

I'm trying to develop a program in Python that allows the user to import datasets and perform operations on them using pandas and numpy so she/he can skip writing all the preprocessing code her/himself.

I have two questions. Am I using my functions poorly? For example, can I create cleaner, more efficient code by using more classes and creating objects with def SomeCode(self, etc, etc)? I still struggle with understanding the concept of __init__ and self because I mainly use Python to clean and analyze data. I'm worried I'm writing some real spaghetti code.

Second, I need to save the state of the data frame after it's been operated on. For example, at the beginning of the program, the user imports a dataset and it's read into the variable df. The user then has options to perform operations on the dataset, such as drop columns by index, rename columns, etc. After a user operates on the data frame, I need to save the new state of the data frame and have it reflected everywhere else in the program. So the next time the user displays the dataset, he sees the dataset with the changes. For example, if the user re-displays the columns, the columns he dropped earlier are not displayed.

class Clean:

# Imports file, displays some information about the dataset
def Main():
    while True:
        file_name = input("\nPlease enter the name of the .csv file you want to process (if the .csv file is not in the same directory as this program, please provide relative path): \n")
        try:
            print("Reading in file...\n")
            df = pd.read_csv(file_name) # Reads in file and stores it in df variable
            df_types = (df.dtypes) # Reads in data types of dataset
            df_columns = (df.columns) # Reads in columns of dataset
            df_shape = (df.shape) # Reads in the 'shape' or dimensions of dataset
            df_null_values = (df.isnull().sum(axis=0)) # Reads in the counts of null values in columns

            # Prints information to screen
            print("Here is some information about your dataframe...\n")
            time.sleep(.5)
            print("Your data types: \n\n{}".format(df_types))
            time.sleep(.5)
            print("\nYour column names:\n {}".format(df_columns))
            time.sleep(.5)
            print("\nThe shape of your dataframe is {}\n".format(df_shape))
            time.sleep(.5)
            print("Here is the count of null values per column:\n{}\n".format(df_null_values))

        except (FileNotFoundError):
            print("File path does not exist. Please enter valid path.\n")
        else:
            break

    # Ran when user types in "exit" at any point in the program
    def ExitProgram():
            double_check = input("Are you sure you want to exit the program? (yes/no) (NOTE: Saying 'no' will return you to option menu.)\n")
            if double_check in yes_values:
                print("\nThanks for cleaning your dirty data!")
                exit()
            elif double_check in no_values:
                DoNext()

    def SaveDataframeState(temp_df):
        temp_file_name = "temp.csv"
        temp_df.to_csv(temp_file_name)
        df = pd.read_csv(temp_file_name)

    # Hashes columns to an index
    def ColumnsToIndex():
        column_list = []
        index_of_list = []

        for col in df_columns:
            column_list.append(col)
            length_of_list = len(column_list)
        for num in range(length_of_list):
            index_of_list.append(num)
        hash = {k:v for k, v in zip(index_of_list, column_list)}
        print("\nHere is the index of columns...\n")
        for k,v in hash.items():
            print(k, ":", v)
    ColumnsToIndex()

    # Displays the amount of rows user inputs
    def DisplayInputtedRows():
        while True:
            try:
                rows_to_display = input("\nHow many rows would you like to display? (Note: Whole dataset will have a limited display in terminal)\n")
                time.sleep(.5)
                print(df.head(int(rows_to_display))) # prints inputted rows to screen
            except (ValueError):
                print("Please pass an integer.")
            else:
                break
    DisplayInputtedRows()

    # Displays the amount of rows user inputs when they type 'row' on option menu
    def RedisplayRows():
        while True:
            try:
                rows_to_redisplay = input("\nHow many rows would you like to display? (Note: Whole dataset will have a limited display in terminal)\n")
                time.sleep(.5)
                print(df.head(int(rows_to_redisplay)))
                DoNext()
            except (ValueError):
                print("Please pass an integer.")
            else:
                break

    def RenameColumns():
        print("\nHere are your columns by name:\n{}".format(df_columns))
        rename_columns = input("\nWhat columns would you like to rename?\n")
        print(rename_columns)
        if rename_columns == "return":
            DoNext()
        elif rename_columns == 'exit':
            ExitProgram()

    def DropOneColByIndex():
        drop_one_col_by_index = input("What columns do you want to drop? Please type in the index:\n")
        temp_df = df.drop(df.columns[int(drop_one_col_by_index)], axis=1)
        print(temp_df.head())
        DoNext()

    def DropColumnsByIndexPrompt():
        print("Here are your columns by name:\n{}".format(df_columns))
        ColumnsToIndex()

        drop_more_than_one_col = input("\nWould you like to drop ONLY 1 COLUMN? Types 'yes' to ONLY drop 1 COLUMN, type 'no' to drop MORE THAN 1 COLUMN.\n")

        if drop_more_than_one_col in yes_values:
            DropOneColByIndex()
        elif drop_more_than_one_col in no_values:
            print()
        elif drop_more_than_one_col == "return":
            DoNext()
        elif drop_more_than_one_col== 'exit':
            ExitProgram()

    time.sleep(.5)

    def DropColumnsByDatatype():
        print("Here are your columns by data-type:\n{}".format(df_types))
        drop_columns_by_datatype = input("\nWhat columns would you like to drop?\n")
        if drop_columns_by_datatype == "return":
            DoNext()
        elif drop_columns_by_datatype == 'exit':
            ExitProgram()

    # Main option screen where user can select operations on dataframe
    def DoNext():
        print("\n(NOTE: If at any point in the program you want to exit back to this option menu, just type 'return'.)\n")
        print("(NOTE: At this part of the program you can also redisplay rows by typing 'rows'.)\n")

        do_next = input("\nWhat would you like to do next?\n[0] Rename Columns \n[1] Drop Column(s) by Index \n[2] Drop Column(s) by Data-type\n")
        if do_next == '0':
            RenameColumns()
        elif do_next == '1':
            DropColumnsByIndexPrompt()
        elif do_next == '2':
            DropColumnsByDatatype()
        elif do_next == 'exit':
            ExitProgram()
        elif do_next == "rows":
            RedisplayRows()
    DoNext()

Main()
\$\endgroup\$
3
\$\begingroup\$

First, Python has an official style-guide, PEP8, which programmers are encouraged to follow. It recommends using lower_case for variables and functions and PascalCase only for classes.

Next, currently your code is just a big mess of functions being defined, sometimes only within some inner scope, sometimes being called directly after being defined and sometimes not. I think making this into a proper class would be a good idea.

Ideally this class will hold the dataframe object, have a method to construct the object with a given filename, have all those functions you designed as methods and give access to the underlying dataframe in case you need to do something not implemented (this would be already given by being able to access self.df, but we can do better).

class DataFrameHelper:
    @classmethod
    def from_file(cls, file_name):
        df = pd.read_csv(file_name)
        return cls(df)

    def __init__(self, df):
        self.df = df

    def __repr__(self):
        return repr(self.df)

    def __str__(self):
        return str(self.df)

    def __getattribute__(self, name):
        try:
            return super().__getattribute__(name)
        except AttributeError:
            return getattr(self.df, name)

    ...

This is all that is needed for the setup. When initializing this class you either need to pass in a dataframe (DataFrameHelper(df)), or use the classmethod which returns an instance of the class when given a file name (DataFrameHelper.from_file(file_name)).

The __repr__ magic method gets called whenever you just type x in an interactive session (with x being an instance of this class). Similarly __str__ gets called when you do print(x).

The __getattribute__ magic method will be called whenever we try to access an attribute of the instance (e.g. x.dtypes). It first tries to find that attribute in the class (so that methods we define there have priority) and if that fails tries to find the attribute in the dataframe. The latter call might still fail, in which case it will just give back that error. This way we don't need your df_dtypes, df_columns and df_shape variables anymore.

Now we just need to add your methods to this class:

    ...

    @property
    def null_values(self):
        return self.isnull().sum(axis=0)

    def save_data(self, file_name="temp.csv"):
        self.df.to_csv(file_name)

    def columns_to_index(self):
        print("\nHere is the index of columns...\n")
        for k, name in enumerate(self.columns):
            print(k, ":", name)

    def display_n_rows(self):
        n = ask_user("\nHow many rows would you like to display? (Note: Whole dataset will have a limited display in terminal)\n", int)
        print(self.head(n))

    def rename_columns(self):
        print("\nHere are your columns by name:")
        print(self.columns)
        rename_columns = input("\nWhat columns would you like to rename?\n")
        if rename_columns == "return":
            return
        elif rename_columns == 'exit':
            EXIT()
        else:
            raise NotImplementedError

    def drop_column_by_index(self):
        i = ask_user("What column do you want to drop? Please type in the index:\n", int)
        self.df.drop(self.columns[i], axis=1, inplace=True)
        print(self.head())

    def drop_columns_by_index(self):
        print("Here are your columns by name:")
        print(self.columns)
        self.columns_to_index()

        n_cols = input("\nWould you like to drop ONLY 1 COLUMN? Types 'yes' to ONLY drop 1 COLUMN, type 'no' to drop MORE THAN 1 COLUMN.\n")

        if n_cols in yes_values:
            self.drop_column_by_index()
        elif n_cols in no_values:
            raise NotImplementedError
        elif n_cols == "return":
            return
        elif n_cols== 'exit':
            EXIT()

    def drop_columns_by_type(self):
        print("Here are your columns by data-type:\n")
        print(self.dtypes)
        dtype = input("\nWhat columns would you like to drop?\n")
        if dtype == "return":
            return
        elif dtype == 'exit':
            EXIT()
        else:
            raise NotImplementedError

And finally we just need to add the menu around it and the two functions I added above, ask_user, which asks the user until an answer is given that can be cast to the given type and passes an optional validator, and EXIT:

import os
import pandas as pd
import sys

def ask_user(message, type_=str, validator=lambda x: True, invalid="Not valid"):
    while True:
        try:
            x = type_(input(message))
            if validator(x):
                return x
            else:
                print(invalid)
        except ValueError:
            print("Please pass a(n)", type_)

def EXIT():
    double_check = input("Are you sure you want to exit the program? (yes/no) (NOTE: Saying 'no' will return you to option menu.)\n")
    if double_check in yes_values:
        print("\nThanks for cleaning your dirty data!")
        sys.exit()

def main():
    file_name = ask_user("\nPlease enter the name of the .csv file you want to process (if the .csv file is not in the same directory as this program, please provide relative path): \n",
                         validator=os.path.isfile,
                         invalid="File path does not exist. Please enter valid path.\n")
    df = DataFrameHelper.from_file(file_name)
    print("Here is some information about your dataframe...\n")
    print("Your data types:\n\n", df.dtypes)
    print("\nYour column names:\n", df.columns)
    print("\nThe shape of your dataframe is {}\n".format(df.shape))
    print("Here is the count of null values per column:\n{}\n".format(df.null_values))

    print("\n(NOTE: If at any point in the program you want to exit back to this option menu, just type 'return'.)\n")

    while True:
        do_next = input("\nWhat would you like to do next?\n[0] Rename Columns \n[1] Drop Column(s) by Index \n[2] Drop Column(s) by Data-type\n")
        if do_next == '0':
            df.rename_columns()
        elif do_next == '1':
            df.drop_columns_by_index()
        elif do_next == '2':
            df.drop_columns_by_type()
        elif do_next == 'exit':
            EXIT()
        elif do_next == "rows":
            df.display_n_rows()

if __name__ == "__main__":
    main()

NB: I got rid of all of your sleep(0.5) calls. Don't let the user wait just so it seems that your program is doing something.

\$\endgroup\$
  • \$\begingroup\$ Wow, this not only helped me organize my code better but also gave me an understanding of things I was having a hard time grasping. To comment on the "sleep" function, it's there so the user is not overwhelmed by all the data getting printed to the screen. \$\endgroup\$ – Dave Guerrero Feb 22 at 18:43

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.