1
\$\begingroup\$

My jupyter notebook is here. I would love to hear any feedback about any problems that may be occurring in my data pipeline.

I already know that I still need to develop the following features:

  1. Implement better hyperparameter testing for the various estimators that I am looking at, especially the linear and SVR regressors which are currently not being cross-validated in my model.

  2. Write functions that represent predictions and importance analysis in a visually understandable way (and using real values instead of the normalized values that are actually being used in the analysis).

    # Get the Data
    
    import pandas as pd
    from matplotlib import pyplot as plt
    import numpy as np
    
    def read_csv(data_file='data.csv'):
        with open(data_file, 'r') as infile:
            dataframe = pd.read_csv(infile)
        return dataframe
    
    df = read_csv()
    
    # Convert numeric data stored in strings to ints or floats,
    # using a dict to parse letter grades.
    
    def convert_to_numeric(dataframe):    
        convert_to_numeric = ['A&I Academics', 'Michigan Placement', 'Arranging 1 ESPA Score', 
                              'Arranging 2 ESPA Score', 'Harmony 1 ESPA Score', 'Harmony 2 ESPA Score', 
                              'Harmony 3 ESPA Score', 'Ear Training ESPA Score', 'First Semester SAP GPA', 
                              'First Semester SAP Credits CUM Attempted', 
                              'First Semester SAP Credits CUM Completed', 
                              'First Semester SAP CUM Credit Completion %',
                              'A&I Reading', 'A&I Standardized Reading', 'A&I Instrumental', 
                              'A&I Standardized Instrumental', 'A&I Scholarship', 
                              'A&I Standardized Scholarship']
        for field in convert_to_numeric:
            dataframe[field] = pd.to_numeric(dataframe[field], errors='coerce')
        letter_grade_fields = ['Harmony Grade', 'Ear Training Grade', 'Arranging Grade']
        conv_dict = {'A': 9., 'A-': 8., 'B+': 7., 'B': 6.,
                     'B-': 5., 'C+': 4., 'C': 3., 'C-': 2.,
                     'D': 1., 'F': 0., 'I': 0., 'IF': 0.,
                     'NG': 0., 'NO INFO': 0., 'W': 0., }
        for field in letter_grade_fields:
            dataframe[field] = dataframe[field].apply(conv_dict.get)
        return dataframe
    
    # Drop certain fields that contain "after the fact" data 
    # such as semester grades. 'Arranging Grade' is also not 
    # used, because it is only populated for students who placed 
    # out of PW-111.
    
    def drop_fields(dataframe):
        to_drop = ['Swiped Y N', 'First Semester SAP GPA', 'Arranging Grade', 
                   'First Semester SAP Credits CUM Completed', 
                   'First Semester SAP Credits CUM Attempted', 
                   'First Semester SAP CUM Credit Completion %']
        return dataframe.drop(to_drop, axis=1)
    
    # Include only students who placed into level 1 of the respective classes.
    
    def get_hr_et_data(dataframe):
        harmony = dataframe[dataframe['Harmony Placement'] == 'Music Application and Theory']
        ear_training = dataframe[dataframe['Ear Training Placement'] == 'Ear Training 1']
        return harmony, ear_training
    
    # Include only numeric data.
    
    def drop_non_numeric(dataframe):
        return dataframe.select_dtypes(include=['float64', 'int64'])
    
    def process_all(dataframe):
        dataframe = convert_to_numeric(dataframe)
        dataframe = drop_fields(dataframe)
        harmony_dataframe, ear_training_dataframe = get_hr_et_data(dataframe)
        harmony_dataframe = drop_non_numeric(harmony_dataframe)
        ear_training_dataframe = drop_non_numeric(ear_training_dataframe)
        return harmony_dataframe, ear_training_dataframe
    
    harmony_df, ear_training_df = process_all(df)
    
    # Split the data into a training and a test set, 
    # using hashes so that this splitting is performed 
    # in a consistent way every time this program is run.
    
    import hashlib
    
    def test_set_check(identifier, test_ratio, hash):
        # Next time, just use sklearn.model_selection.train_test_split()
        return hash(np.int64(identifier)).digest()[-1] < 256 * test_ratio
    
    def split_train_test_by_id(data, id_column, test_ratio=.2, hash=hashlib.md5):
        ids = data[id_column]
        in_test_set = ids.apply(lambda id_: test_set_check(id_, test_ratio, hash))
        return data.loc[~in_test_set], data.loc[in_test_set]
    
    hr_train_set, hr_test_set = split_train_test_by_id(harmony_df, "Student #")
    et_train_set, et_test_set = split_train_test_by_id(ear_training_df, "Student #")
    
    # Take a look at the data.
    
    #hr_corr_matrix = hr_train_set.corr()
    #hr_corr_matrix['Harmony Grade'].sort_values(ascending=False)
    
    #et_corr_matrix = et_train_set.corr()
    #et_corr_matrix['Ear Training Grade'].sort_values(ascending=False)
    
    # Prepare the data for machine learning algorithms.
    
    harmony = hr_train_set.drop('Harmony Grade', axis=1)
    harmony_labels = hr_train_set['Harmony Grade'].copy()
    
    ear_training = et_train_set.drop('Ear Training Grade', axis=1)
    ear_training_labels = et_train_set['Ear Training Grade'].copy()
    
    # Handle the numeric data:
    # Use Imputer to supply a median value where 
    # a value is missing, and StandardScaler to 
    # normalize the values.
    
    from sklearn.preprocessing import Imputer, StandardScaler
    from sklearn.pipeline import Pipeline
    from sklearn.base import BaseEstimator, TransformerMixin
    
    # Select the data from the pandas dataframe.
    
    class DataFrameSelector(BaseEstimator, TransformerMixin):
        def __init__(self):
            return None
        def fit(self, X, y=None):
            return self
        def transform(self, X):
            return X.values
    
    num_pipeline = Pipeline([
        ('dataframe_selector', DataFrameSelector()),
        ('imputer', Imputer(strategy='median')),
        ('std_scaler', StandardScaler()),
    ])
    
    harmony_tr = num_pipeline.fit_transform(harmony)
    ear_training_tr = num_pipeline.fit_transform(ear_training)
    
    # Import various estimators, and create functions to test 
    # them and their hyperparameters.
    
    import os
    import csv
    
    from sklearn.linear_model import LinearRegression, LassoCV, ElasticNetCV, RidgeCV
    from sklearn.tree import DecisionTreeRegressor
    from sklearn.ensemble import RandomForestRegressor
    from sklearn.svm import SVR
    from sklearn.model_selection import cross_val_score
    from sklearn.model_selection import GridSearchCV
    
    def test_model(name, data, labels, model, verbose=False):
        scores = cross_val_score(model, data, labels, 
                                 scoring='neg_mean_squared_error', cv=10)
        rmse_scores = np.sqrt(-scores)
        if verbose:
            print('### {} Results ###'.format(name))
            print("Scores:", scores)
            print("Mean:", rmse_scores.mean())
            print("Standard deviation:", rmse_scores.std())
        return rmse_scores
    
    def implement_model(name, data, labels, model, param_grid):
        if params:
            grid_search = GridSearchCV(model, param_grid, cv=5,
                                   scoring='neg_mean_squared_error')
            grid_search.fit(data, labels)
            estimator = grid_search.best_estimator_
        else:
            model.fit(data, labels)
            estimator = model
        scores = test_model(name, data, labels, estimator)
        with open('results.csv', 'a') as outfile:
            writer = csv.writer(outfile)
            writer.writerow([name, str(estimator), scores.mean(), scores.std()])
        return estimator
    
    def implement_hr_et(et_data, hr_data, et_labels, hr_labels, model, param_grid):
        hr_model = implement_model('Harmony', hr_data, hr_labels, model, param_grid)
        et_model = implement_model('Ear Training', et_data, et_labels, model, param_grid)
        return hr_model, et_model
    
    # Make a file to store the results of testing.
    
    if os.path.isfile('results.csv'):
        os.remove('results.csv')
        with open('results.csv', 'a') as outfile:
            writer = csv.writer(outfile)
            writer.writerow(['Dataset', 'Model & Best Params', 'Score', 'STERR'])
    
    # Test various estimators and parameters. If params is 
    # populated, then we will cross-validate using them GridSearchCV 
    # to determine the best parameters; however some estimators do this 
    # automatically (e.g. LassoCV) and others do not require it 
    # because of how they work out of the box (e.g. RandomForestRegressor).
    
    # LINEAR REGRESSOR
    
    params = [] # TODO
    hr_linear, et_linear= implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
                    harmony_labels, LinearRegression(), params)
    
    # DECISION TREE REGRESSOR
    
    params = [
              {'max_features': [2, 3, 4, 6, 8, 10, 12, 16, 18]},
             ]
    hr_tree, et_tree = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
                    harmony_labels, DecisionTreeRegressor(), params)
    
    # RANDOM FOREST REGRESSOR
    
    params = []
    hr_forest, et_forest = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
                    harmony_labels, RandomForestRegressor(), params)
    
    # LASSO
    
    params = []
    hr_lasso, et_lasso = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
                    harmony_labels, LassoCV(), params)
    
    # RIDGE
    
    params = []
    hr_ridge, et_ridge = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
                    harmony_labels, RidgeCV(), params)
    
    # ELASTIC NET
    
    params = []
    hr_enet, et_enet = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
                    harmony_labels, ElasticNetCV(), params)
    
    # SVR
    
    params = [] # TODO
    hr_svr, et_svr = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
                    harmony_labels, SVR(kernel='linear'), params)
    
    # Display Harmony feature importances:
    
    hr_importances = hr_tree.feature_importances_
    sorted(zip(hr_importances, harmony.columns), reverse=True)
    
    # Display Ear Training feature importances:
    
    et_importances = et_tree.feature_importances_
    sorted(zip(et_importances, ear_training.columns), reverse=True)
    
    from sklearn.tree import export_graphviz
    with open("et_dtree.dot", 'w') as outfile:
        export_graphviz(et_tree, out_file = outfile, feature_names = ear_training.columns, 
                       max_depth=3)
    os.system("dot -Tpng et_dtree.dot -o et_dtree.png")
    os.remove('et_dtree.dot')
    

Thanks!

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.