Ok , for my final year project I've wrote this piece of code to train my machine learning model on a this dataset , here the code i used

import pandas as pd
import numpy as np
from sklearn.linear_model import Ridge,Lasso,ElasticNet,LinearRegression
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.svm import LinearSVR,SVR
from sklearn.metrics import mean_squared_error
from sklearn.externals import joblib
from sklearn.preprocessing import LabelBinarizer

class PredictiveModelBuilding(object):

    this class will handle all pipeline for a preeictive modele building
    in the chapter of my machine learning project ,
    it will train differents models, encode data,scale data, and predict
    def __init__(self, dataset, encoderFunction):

        dataframe : pandas dataframe
        this if the dataframe we will use to train the model
        encoderFunction : function
        the function used to encode categorical data in the dataframe

        the dataset:
        training set : the training set of our dataset
        test_set : the test set
        preidictive_models : the models we will use to predict the new
        stacker : the satcker of our 5 predictives models
        if isinstance(dataset, pd.DataFrame):
            self.dataset = dataset
            self.training_set = pd.DataFrame()
            self.test_set = pd.DataFrame()
            self.predictive_models = {}
            self.x_train = pd.DataFrame()
            self.x_test = pd.DataFrame()
            self.y_train = pd.Series()
            self.y_test = pd.Series()
            self.dataset_bin, self.encoders = encoderFunction(dataset, cat_col=['SCHOOL_RIGHT', 'OPTION_RIGHT'], num_col=['DIPPERC', 'CGPA', 'EchecRatio'])
            ridge_reg = Ridge(alpha=1, solver="cholesky", fit_intercept=False, max_iter=20000)
            linSVM_reg = LinearSVR(dual=False, fit_intercept=False,loss='squared_epsilon_insensitive', max_iter=20000)
            rbfSVM_reg = SVR(verbose=True, max_iter=20000)
            lasso_reg = Lasso(alpha=1e-05, max_iter=20000, fit_intercept=False)
            elastic_reg = ElasticNet(alpha=1e-05, max_iter=20000, l1_ratio=0.5)
            self.predictive_models[ridge_reg.__class__.__name__] = ridge_reg
            self.predictive_models[linSVM_reg.__class__.__name__] = linSVM_reg
            self.predictive_models[rbfSVM_reg.__class__.__name__] = rbfSVM_reg
            self.predictive_models[lasso_reg.__class__.__name__] = lasso_reg
            self.predictive_models[elastic_reg.__class__.__name__] = elastic_reg
            self.stacker = LinearRegression(normalize=True)
            raise TypeError('need only a DataFrame')

    def scale(self, num_cols):

        this function will scale the values of GPA and DIP percentage
        by divide them by 100

        self.dataset_bin.loc[:, num_cols[0]] = self.dataset_bin[num_cols[0]]/100
        self.dataset_bin.loc[:, num_cols[1]] = self.dataset_bin[num_cols[1]]/100

    def split(self):

        the function will split the dataset into a train and a test one
         and return x_train and x_Test
        the dataset:
        training set : the training set description
        and the test set descriptions

        split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=4)
        for train_index, test_index in split.split(self.dataset_bin, self.dataset_bin.EchecRatio):
            self.training_set = self.dataset_bin.loc[train_index]
            self.test_set = self.dataset_bin.loc[test_index]
        self.training_set.set_index(keys='ID', inplace=True)
        self.test_set.set_index(keys='ID', inplace=True)
        self.y_train = self.training_set.CGPA
        self.x_train = self.training_set.drop(labels=['CGPA', 'EchecRatio'], axis=1)
        self.y_test = self.test_set.CGPA
        self.x_test = self.test_set.drop(labels=['CGPA', 'EchecRatio'], axis=1)
        train_descrption = self.training_set.describe()[['DIPPERC', 'CGPA']]
        test_description = self.test_set.describe()[['DIPPERC', 'CGPA']]
        return train_descrption, test_description

    def train(self):

        this  will fit our predictives models with x_train and
        y_train, and make predictions with them
        the function will return the dataset of predicted values
        predictions = {}
        for clf in self.predictive_models.values():
            clf.fit(self.x_train, self.y_train)
            predictions[clf.__class__.__name__] = clf.predict(self.x_train)
        predicted_values = pd.DataFrame.from_dict(predictions, dtype=np.float)
        predicted_values.set_index(self.y_train.index, inplace=True)
        predicted_values.loc[:, 'RealValue'] = self.y_train
        return predicted_values

    def predict_test(self):

        this  will fit our predictives models with x_test and
        y_test, and make predictions with them
        the function will return the dataset of predicted values


        predictions = {}
        for clf in self.predictive_models.values():
            predictions[clf.__class__.__name__] = clf.predict(self.x_test)
        predicted_values = pd.DataFrame.from_dict(predictions, dtype=np.float)
        predicted_values.set_index(self.y_test.index, inplace=True)
        predicted_values.loc[:, 'RealValue'] = self.y_test
        return predicted_values

    def predict_new(self, new_student_data):

        the function will handle predicting data on new data

        the data will be received as json object and it will be encoded
        new_student_data : the new student data as a dict
        Description of arg1

        the final predicted value


        option_encoder = self.encoders['OPTION_RIGHT']
        school_encoder = self.encoders['SCHOOL_RIGHT']
        options = pd.DataFrame(
        data=dict(zip(option_encoder.classes_, option_encoder.transform(new_student_data[['OPTION_RIGHT']])[0])),
            index=new_student_data.index, columns=option_encoder.classes_)
        schools = pd.DataFrame(data=dict(zip(school_encoder.classes_, school_encoder.transform(new_student_data[['SCHOOL_RIGHT']])[0])), index=new_student_data.index, columns=school_encoder.classes_)
        new_dataset = pd.merge(options, schools, on='index')
        new_dataset['DIPPERC'] = new_student_data['DIPPERC']
        new_dataset.set_index(keys=['index'], inplace=True)
        predictions = {}
        for clf in self.predictive_models.values():
            predictions[clf.__class__.__name__] = clf.predict(new_dataset)
        predicted_values = pd.DataFrame.from_dict(predictions, dtype=np.float)
        predicted_values.set_index(new_dataset.index, inplace=True)
        predicted_values.loc[:, 'finalOutput'] = self.stacker.predict(predicted_values)
        return predicted_values['finalOutput']

    def evaluate(self, model, sur):


        this function will first do a evaluation of a model and return
        the rmse score of it and some data and their labels the function
        can evaluate on the training set  and also on test_set

        sur : string
        [test,or train] to tell on which dataframe the model will be
        model : predictive model
        the model to evaluate

        teh root mean square error

        if sur == 'train':
            some_data = self.x_train.iloc[:5]
            some_labels = self.y_train.iloc[:5]
            print ("Predictions:\t", self.predictive_models[model].predict(some_data))
            print ("Labels:\t\t", list(some_labels))
            cgpa_predictions = self.predictive_models[model].predict(self.x_train)
            lin_mse = mean_squared_error(self.y_train, cgpa_predictions)
            lin_rmse = np.sqrt(lin_mse)
            return lin_rmse
        elif sur == 'test':
            some_data = self.x_test.iloc[:5]
            some_labels = self.y_test.iloc[:5]
            print ("Predictions:\t", self.predictive_models[model].predict(some_data))
            print ("Labels:\t\t", list(some_labels))
            cgpa_predictions = self.predictive_models[model].predict(self.x_test)
            lin_mse = mean_squared_error(self.y_test, cgpa_predictions)
            lin_rmse = np.sqrt(lin_mse)
            return lin_rmse

    def cross_evaluate(self, model):

        this one will perfom a cross validation of the model

        model : predictive model
        the model to evaluate

        RMSE_SCORE : tuple
        teh root mean square errors, theirs mean, and std deviation

        scores = cross_val_score(self.predictive_models[model], self.x_train, self.y_train, scoring="neg_mean_squared_error", cv=10)
        rmse_scores = np.sqrt(-scores)
        return rmse_scores, rmse_scores.std(), rmse_scores.mean()

    def ensemble_methods(self, predicted_values):

        this method will get a dataframe of predicted values by diffrents classifier and will return
        the value compute by  a linear regression between the 3 values and rmse

        predicted_values : pandas dataframe : the dataframe of
        predicted values

        predicted_values : the final results
        rmse_ensemble : the root mean square error of ensemble models

        labels = ['ElasticNet', 'Lasso', 'LinearSVR', 'Ridge', 'SVR']
        x_new = predicted_values[labels]
        y_new = predicted_values.RealValue
        self.stacker.fit(x_new, y_new)
        final_predict = self.stacker.predict(x_new)
        predicted_values.loc[:, 'finalPredict'] = final_predict
        rmse_ensemble = np.sqrt(mean_squared_error(y_new, final_predict))
        return predicted_values, rmse_ensemble

    def save_models(self, departement):

        after all job we will save the class with the models for

            joblib.dump(self, "../GradePredictorApp/app/static/classes/"+departement+".pkl")
        except Exception as exception:
            raise TypeError("model floder not found "+str(exception))
            joblib.dump(self, departement+".pkl")

def convert_cat(dataset, cat_col, num_col):

    this function will binarize a dataset given in parametrer and
    return the dataset with categorical columns binarise by one-hot

    encs = {}
    x_train_1 = dataset[cat_col]
    x_new = dataset[num_col]
    cat_col = x_train_1.columns
    for col in cat_col:
        data = dataset[[col]]
        enc = LabelBinarizer()
        # Fitting One Hot Encoding on train data
        temp = enc.transform(dataset[[col]])
        # Changing the encoded features into a data frame with new column names
        temp = pd.DataFrame(temp, columns=enc.classes_)
        # In side by side concatenation index values should be same
        # Setting the index values similar to the X_train data frame
        temp = temp.set_index(dataset.index)
        # adding the new One Hot Encoded varibales to the train data frame
        x_new = pd.merge(temp, x_new, right_index=True, left_index=True)
        #saving the encoder into a dict for others operations
        encs[col] = enc
    return x_new, encs

def final_job(dataframe):

    this method will handle all the task related to training models in
    each departement and in final they will return a dataset with
    all types of erros .

    #first we iterate over the whole dataset to get each departement

    results = {}
    predicted_resuts = {}
    for departement, datas in dataframe.groupby('FAC'):
        results[departement] = []
        predictive_model = PredictiveModelBuilding(dataset=datas, encoderFunction=convert_cat)
        predictive_model.scale(['DIPPERC', 'CGPA'])
        train_des, test_des = predictive_model.split()
        print train_des
        print test_des
        predicted_values = predictive_model.train() #trainig the models
        rmse = {}
        for name, model in predictive_model.predictive_models.items():
            cgpa_mean = predictive_model.dataset_bin.CGPA.mean()
            rmse_train = predictive_model.evaluate(model=name, sur='train') #rmse of each model
            rmse[name] = [rmse_train, rmse_train*100/cgpa_mean]
            scores, score_std, score_mean = predictive_model.cross_evaluate(model=name)
            cv_score = [score_mean, score_mean*100/cgpa_mean, score_std]
            print (model, scores)
            rmse_test = predictive_model.evaluate(model=name, sur='test') #rmse of each model
            rmse[name].append([rmse_test, rmse_test*100/cgpa_mean])
        final_predict, final_rmse = predictive_model.ensemble_methods(predicted_values)
        print final_predict.head(5)
        new_student = {'DIPPERC':0.60, 'SCHOOL_RIGHT':'itfm/bukavu', 'OPTION_RIGHT':'elec indust'}
        new_student_data = pd.DataFrame(new_student, columns=new_student.keys(), index=range(1))
        predicted_resuts[departement] = predictive_model.predict_new(new_student_data)
        results[departement].append([final_rmse, final_rmse*100/cgpa_mean])
    return results, predicted_resuts

def build_final_dataset(results):

    this methods will help us to build the final dataset for our report

    results_tab = pd.DataFrame(columns=[0, 1, 2, 3, 'CVSCORE Mean', u'CVSCORE Std', u'RMSE Train', u'RMSE Test'], index=range(0, 7))
    results_tab_2 = pd.DataFrame(
        columns=['STACK_RES', 'Dimensions', 'Faculte'],
        index=range(0, 35))
    final_dataframes = []
    next_index = 2 #index where we want to put the values in tab_2
    for name , val in results.items():
        results_data = pd.DataFrame.from_dict(val[1], orient='index')
        cv_score_mean = []
        cv_score_std = []
        for ind in results_data[2].index:
            cv_mean = (str("%.3f" % results_data[2][ind][0])) + "  : " + str("%.2f" % results_data[2][ind][1]) + '%'
            cv_score_std.append("%.4f" % results_data[2][ind][2])

        results_data['CVSCORE Mean'] = cv_score_mean
        results_data['CVSCORE Std'] = cv_score_std
        rmse_score_mean = []

        for ind in results_data[[0, 1]].index:
            rmse_mean = (str("%.2f" % results_data [0][ind])) + " : " + str("%.2f" % results_data [1][ind]) + '%'
        results_data['RMSE Train'] = rmse_score_mean
        rmse_score_mean_test = []

        for ind in results_data[3].index:
            rmse_mean = (str("%.2f" % results_data [3][ind][0])) + " : " + str("%.3f" % results_data [3][ind][1]) + '%'
        results_data['RMSE Test'] = rmse_score_mean_test
        final_score = (str("%.3f" % results.get(name)[2][0])) + " : " + str("%.3f" % results.get(name)[2][1]) + '%'

        results_tab_2.STACK_RES[next_index] = final_score
        results_tab_2.Dimensions[next_index] = str(results.get(name)[0])
        results_tab_2.Faculte[next_index] = name
        final_dataframes.append(results_data[[u'CVSCORE Mean', u'CVSCORE Std', u'RMSE Train', u'RMSE Test']])
        next_index += 5
    results_data = pd.concat(final_dataframes)
    results_data.rename(columns={'index':'MODEL'}, inplace=True)
    final = pd.merge(results_tab_2, results_data, on='index')
    return final

Now , i have to show this code to a guy with 20 years of experience in programming for a junior developer job .

I want first to know if it is presentable or showable , also if it is readable and some performance issue on how i work with pandas dataframes.

Any Help will be appreciated

if you need more info about the project it can be found on my github


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.