# Stacking and folding machine-learning algorithm

I am trying to use the code of this guy. Written in 2013 but it is a bit obsolete on some points. Basically, the idea is to create folders from 1 dataset to train several models independently on the same data. Then, I can do a second level training based on the results of the first level. Combining models like this increase the accuracy and efficiency of the predictions.

My main performance issue is the creation of the different datasets in the folders. I am not sure my approach with the for loops is optimum. The time required is really long and I think it can be decreased.

from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
def ModelingTrainingAndCompare(dataframe):

general, unseen = train_test_split(dataframe, test_size=0.2)
scaler = StandardScaler()
X_general=general.drop('Modified',axis=1)
Y_general= general['Modified']
X_unseen=unseen.drop('Modified',axis=1)
Y_unseen=unseen['Modified']

classifiers = [ # optimized models used for stacking

DecisionTreeClassifier(criterion='gini',
max_features=None,
random_state=1,
class_weight=None,
splitter='random',
presort='auto'),
SVC(kernel='linear',
C=0.025),
RandomForestClassifier(n_jobs=-1,
n_estimators=500,
max_features= 'auto',
min_samples_leaf=2,
warm_start=True,
class_weight=None,
random_state=1,
oob_score=True
),
learning_rate=0.01,
algorithm='SAMME',
random_state=None,
base_estimator=ExtraTreesClassifier()
),
ExtraTreesClassifier( criterion='entropy', n_jobs= -1,
n_estimators=500,
max_features= None,
min_samples_leaf= 2,
random_state=None,
warm_start=True,
class_weight=None
)
]

# Cross Validation :
StrKF=StratifiedKFold(5)
skf=list(StrKF.split(X_general,Y_general))

# create the data before attribution
blend_train=np.zeros((X_general.shape[0],len(classifiers))) #number of training data * number of classifiers used
blend_test=np.zeros((X_unseen.shape[0],len(classifiers))) # number of testing data * nb of classifiers

for i,clf in enumerate(classifiers):

print('Training classifier '+str(i))
blend_test_i=np.zeros((X_unseen.shape[0],len(skf))) # do not suppress the double (()), allows to handle
# the 'data type not understood' error
for  j,(train_index,cv_index)in enumerate(skf):

print(len(cv_index))
X_train=pd.DataFrame()
y_train=pd.DataFrame()
X_cv=pd.DataFrame()
y_cv=pd.DataFrame()
#creation of the different datasets to train, main problem in performances
for Tindex in train_index:
print(len(X_general.iloc[Tindex]))
print(Y_general.shape)
X=pd.Series(X_general.iloc[Tindex].values.tolist())

Y=pd.Series(Y_general.iloc[Tindex])
X_train=X_train.append(X,ignore_index=True)
y_train=y_train.append(Y,ignore_index=True)

for Cindex in cv_index:
print(Cindex)
XCv=pd.Series(X_general.iloc[Cindex].values.tolist())
YCv=pd.Series(Y_general.iloc[Cindex])
X_cv=X_cv.append(XCv,ignore_index=True)
y_cv=y_cv.append(YCv,ignore_index=True)

print(X_train.shape)
print(y_train.isnull().sum())
#trainning each model
clf.fit(X_train, y_train)

# filing the train set and test set of the second layer stacking
blend_train[cv_index,j]=clf.predict(X_cv)
blend_test_i[:,j]=clf.predict(X_unseen)
# using the mean of the predictions of the cv set
blend_test[:,i]=blend_test_i.mean(1)
print (blend_train.shape)
print(blend_test.shape)
#below this is the second layer stacking
Stk.stackingComparison(blend_train,Y_general,blend_test,Y_unseen)


EDIT : my code ran for a really long time and throw an error this morning 'single positional indexer is out-of-bounds'. It seems it is due to my dataframe creation row by row (that is told to be inneficient). I resolve the bug and keep you updated
EDIT 2 : My code is running again now i fixed the bug (calling index on the wrong dataset, and i reduced the data size to get results faster