# sklearn request: pipeline for regression analysis on entering student data

My jupyter notebook is here. I would love to hear any feedback about any problems that may be occurring in my data pipeline.

I already know that I still need to develop the following features:

1. Implement better hyperparameter testing for the various estimators that I am looking at, especially the linear and SVR regressors which are currently not being cross-validated in my model.

2. Write functions that represent predictions and importance analysis in a visually understandable way (and using real values instead of the normalized values that are actually being used in the analysis).

# Get the Data

import pandas as pd
from matplotlib import pyplot as plt
import numpy as np

with open(data_file, 'r') as infile:
return dataframe

# Convert numeric data stored in strings to ints or floats,
# using a dict to parse letter grades.

def convert_to_numeric(dataframe):
convert_to_numeric = ['A&I Academics', 'Michigan Placement', 'Arranging 1 ESPA Score',
'Arranging 2 ESPA Score', 'Harmony 1 ESPA Score', 'Harmony 2 ESPA Score',
'Harmony 3 ESPA Score', 'Ear Training ESPA Score', 'First Semester SAP GPA',
'First Semester SAP Credits CUM Attempted',
'First Semester SAP Credits CUM Completed',
'First Semester SAP CUM Credit Completion %',
'A&I Standardized Instrumental', 'A&I Scholarship',
'A&I Standardized Scholarship']
for field in convert_to_numeric:
dataframe[field] = pd.to_numeric(dataframe[field], errors='coerce')
conv_dict = {'A': 9., 'A-': 8., 'B+': 7., 'B': 6.,
'B-': 5., 'C+': 4., 'C': 3., 'C-': 2.,
'D': 1., 'F': 0., 'I': 0., 'IF': 0.,
'NG': 0., 'NO INFO': 0., 'W': 0., }
dataframe[field] = dataframe[field].apply(conv_dict.get)
return dataframe

# Drop certain fields that contain "after the fact" data
# used, because it is only populated for students who placed
# out of PW-111.

def drop_fields(dataframe):
to_drop = ['Swiped Y N', 'First Semester SAP GPA', 'Arranging Grade',
'First Semester SAP Credits CUM Completed',
'First Semester SAP Credits CUM Attempted',
'First Semester SAP CUM Credit Completion %']
return dataframe.drop(to_drop, axis=1)

# Include only students who placed into level 1 of the respective classes.

def get_hr_et_data(dataframe):
harmony = dataframe[dataframe['Harmony Placement'] == 'Music Application and Theory']
ear_training = dataframe[dataframe['Ear Training Placement'] == 'Ear Training 1']
return harmony, ear_training

# Include only numeric data.

def drop_non_numeric(dataframe):
return dataframe.select_dtypes(include=['float64', 'int64'])

def process_all(dataframe):
dataframe = convert_to_numeric(dataframe)
dataframe = drop_fields(dataframe)
harmony_dataframe, ear_training_dataframe = get_hr_et_data(dataframe)
harmony_dataframe = drop_non_numeric(harmony_dataframe)
ear_training_dataframe = drop_non_numeric(ear_training_dataframe)
return harmony_dataframe, ear_training_dataframe

harmony_df, ear_training_df = process_all(df)

# Split the data into a training and a test set,
# using hashes so that this splitting is performed
# in a consistent way every time this program is run.

import hashlib

def test_set_check(identifier, test_ratio, hash):
# Next time, just use sklearn.model_selection.train_test_split()
return hash(np.int64(identifier)).digest()[-1] < 256 * test_ratio

def split_train_test_by_id(data, id_column, test_ratio=.2, hash=hashlib.md5):
ids = data[id_column]
in_test_set = ids.apply(lambda id_: test_set_check(id_, test_ratio, hash))
return data.loc[~in_test_set], data.loc[in_test_set]

hr_train_set, hr_test_set = split_train_test_by_id(harmony_df, "Student #")
et_train_set, et_test_set = split_train_test_by_id(ear_training_df, "Student #")

# Take a look at the data.

#hr_corr_matrix = hr_train_set.corr()

#et_corr_matrix = et_train_set.corr()

# Prepare the data for machine learning algorithms.

ear_training = et_train_set.drop('Ear Training Grade', axis=1)

# Handle the numeric data:
# Use Imputer to supply a median value where
# a value is missing, and StandardScaler to
# normalize the values.

from sklearn.preprocessing import Imputer, StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.base import BaseEstimator, TransformerMixin

# Select the data from the pandas dataframe.

class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self):
return None
def fit(self, X, y=None):
return self
def transform(self, X):
return X.values

num_pipeline = Pipeline([
('dataframe_selector', DataFrameSelector()),
('imputer', Imputer(strategy='median')),
('std_scaler', StandardScaler()),
])

harmony_tr = num_pipeline.fit_transform(harmony)
ear_training_tr = num_pipeline.fit_transform(ear_training)

# Import various estimators, and create functions to test
# them and their hyperparameters.

import os
import csv

from sklearn.linear_model import LinearRegression, LassoCV, ElasticNetCV, RidgeCV
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import SVR
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

def test_model(name, data, labels, model, verbose=False):
scores = cross_val_score(model, data, labels,
scoring='neg_mean_squared_error', cv=10)
rmse_scores = np.sqrt(-scores)
if verbose:
print('### {} Results ###'.format(name))
print("Scores:", scores)
print("Mean:", rmse_scores.mean())
print("Standard deviation:", rmse_scores.std())
return rmse_scores

def implement_model(name, data, labels, model, param_grid):
if params:
grid_search = GridSearchCV(model, param_grid, cv=5,
scoring='neg_mean_squared_error')
grid_search.fit(data, labels)
estimator = grid_search.best_estimator_
else:
model.fit(data, labels)
estimator = model
scores = test_model(name, data, labels, estimator)
with open('results.csv', 'a') as outfile:
writer = csv.writer(outfile)
writer.writerow([name, str(estimator), scores.mean(), scores.std()])
return estimator

def implement_hr_et(et_data, hr_data, et_labels, hr_labels, model, param_grid):
hr_model = implement_model('Harmony', hr_data, hr_labels, model, param_grid)
et_model = implement_model('Ear Training', et_data, et_labels, model, param_grid)
return hr_model, et_model

# Make a file to store the results of testing.

if os.path.isfile('results.csv'):
os.remove('results.csv')
with open('results.csv', 'a') as outfile:
writer = csv.writer(outfile)
writer.writerow(['Dataset', 'Model & Best Params', 'Score', 'STERR'])

# Test various estimators and parameters. If params is
# populated, then we will cross-validate using them GridSearchCV
# to determine the best parameters; however some estimators do this
# automatically (e.g. LassoCV) and others do not require it
# because of how they work out of the box (e.g. RandomForestRegressor).

# LINEAR REGRESSOR

params = [] # TODO
hr_linear, et_linear= implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
harmony_labels, LinearRegression(), params)

# DECISION TREE REGRESSOR

params = [
{'max_features': [2, 3, 4, 6, 8, 10, 12, 16, 18]},
]
hr_tree, et_tree = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
harmony_labels, DecisionTreeRegressor(), params)

# RANDOM FOREST REGRESSOR

params = []
hr_forest, et_forest = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
harmony_labels, RandomForestRegressor(), params)

# LASSO

params = []
hr_lasso, et_lasso = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
harmony_labels, LassoCV(), params)

# RIDGE

params = []
hr_ridge, et_ridge = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
harmony_labels, RidgeCV(), params)

# ELASTIC NET

params = []
hr_enet, et_enet = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
harmony_labels, ElasticNetCV(), params)

# SVR

params = [] # TODO
hr_svr, et_svr = implement_hr_et(ear_training_tr, harmony_tr, ear_training_labels,
harmony_labels, SVR(kernel='linear'), params)

# Display Harmony feature importances:

hr_importances = hr_tree.feature_importances_
sorted(zip(hr_importances, harmony.columns), reverse=True)

# Display Ear Training feature importances:

et_importances = et_tree.feature_importances_
sorted(zip(et_importances, ear_training.columns), reverse=True)

from sklearn.tree import export_graphviz
with open("et_dtree.dot", 'w') as outfile:
export_graphviz(et_tree, out_file = outfile, feature_names = ear_training.columns,
max_depth=3)
os.system("dot -Tpng et_dtree.dot -o et_dtree.png")
os.remove('et_dtree.dot')


Thanks!