I have a set of houses with categorical and numerical data. Later I will have a new house and my goal will be to find the 20 closest houses. The code is working fine, and the result are not so bad but it is **way too long**. With a sample of 10 000 houses it takes 6 minutes, using Python 2.7. My actual dataset is about 100 000 houses. # -*- coding: utf-8 -*- import csv import random import math import operator import numpy as np import pandas as pd, os from sklearn.preprocessing import OneHotEncoder from sklearn.feature_extraction import DictVectorizer from sklearn.preprocessing import StandardScaler from sklearn.linear_model import LinearRegression from unidecode import unidecode import sys import cProfile import re data_pd = pd.read_csv('data.csv') # I delete rows with NA's (to change later) data = data.dropna() # The function I use to deal with categorical data def one_hot_dataframe(data, cols, replace=False): vec = DictVectorizer() mkdict = lambda row: dict((col, row[col]) for col in cols) vecData = pd.DataFrame(vec.fit_transform(data[cols].apply(mkdict, axis=1)).toarray()) vecData.columns = vec.get_feature_names() vecData.index = data.index if replace is True: data = data.drop(cols, axis=1) data = data.join(vecData) return (data, vecData, vec) # New data set with numerical data only data_encode, _, _= one_hot_dataframe(data, ['Type', 'Bord_de_mer', 'Bord_de_plan_deau', 'Campagne', 'Centre_ville', 'Complexe_de_vacances', 'Lac', 'Montagne', 'Plage', 'Riviere', 'Village', 'Ville', 'Acces_haut_debit', 'Acces_internet', 'Climatisation', 'Linge_de_maison_fourni', 'Serviettes_de_bain', 'Wifi', 'Terrasse', 'Veranda_Loggia', 'Balcon', 'Jardin', 'location_non_fumeur', 'fumeurs_acceptes', 'animaux_autorises', 'animaux_non_admis', 'acces_handicape', 'Piscine_commune', 'Piscine_privee', 'Piscine_dinterieur', 'Piscine_chauffee', 'Bain_a_remous', 'Sauna'], replace=True) #convert string to float data = data.convert_objects(convert_numeric=True) # scaled_col the columns to standardize scaled_col = ['Capacity', 'BedRooms', 'Latitude', 'Longitude', 'Size', 'Acces_haut_debit=No', 'Acces_haut_debit=Yes', 'Acces_internet=No', 'Acces_internet=Yes', 'Bain_a_remous=No', 'Bain_a_remous=Yes', 'Balcon=No', 'Balcon=Yes', 'Bord_de_mer=No', 'Bord_de_plan_deau=No', 'Bord_de_plan_deau=Yes', 'Campagne=No', 'Campagne=Yes', 'Centre_ville=No', 'Centre_ville=Yes', 'Climatisation=No', 'Climatisation=Yes', 'Complexe_de_vacances=No', 'Complexe_de_vacances=Yes', 'Jardin=No', 'Jardin=Yes', 'Lac=No', 'Lac=Yes', 'Linge_de_maison_fourni=No', 'Linge_de_maison_fourni=Yes', 'Montagne=No', 'Montagne=Yes', 'Piscine_chauffee=No', 'Piscine_chauffee=Yes', 'Piscine_commune=No', 'Piscine_commune=Yes', 'Piscine_dinterieur=No', 'Piscine_privee=No', 'Piscine_privee=Yes', 'Plage=No', 'Plage=Yes', 'Riviere=No', 'Riviere=Yes', 'Sauna=No', 'Sauna=Yes', 'Serviettes_de_bain=No', 'Serviettes_de_bain=Yes', 'Terrasse=No', 'Terrasse=Yes', 'Type=Appartement', 'Type=Chalet', "Type=Chambre d'h\xc3\xb4tes", 'Type=G\xc3\xaete', 'Type=Maison', 'Type=Studio', 'Type=Villa', 'Veranda_Loggia=No', 'Veranda_Loggia=Yes', 'Village=No', 'Village=Yes', 'Ville=No', 'Ville=Yes', 'Wifi=No', 'Wifi=Yes', 'acces_handicape=No', 'acces_handicape=Yes', 'animaux_autorises=No', 'animaux_autorises=Yes', 'animaux_non_admis=No', 'animaux_non_admis=Yes', 'fumeurs_acceptes=No', 'fumeurs_acceptes=Yes', 'location_non_fumeur=No', 'location_non_fumeur=Yes'] # standardization of the dataset scaled_features = data.copy() features = scaled_features[scaled_col] scaler = StandardScaler().fit(features.values) features = scaler.transform(features.values) scaled_features[scaled_col] = features # That's the part where I find weights for the knn algorithm # I'm using a simple regression and I keep the parameters #create a new dataframe with the concerned features weightframe = scaled_features weightframe = weightframe.drop('DetailUrl', axis = 1) weightframe = weightframe.drop('index', axis = 1) X = weightframe.drop('Moyenne', axis = 1) y = weightframe['Moyenne'] lm = LinearRegression(fit_intercept=False) lm.fit(X,y) # df with the features and their weights weights = pd.DataFrame(zip(X.columns, lm.coef_)) # df with positive values weights_abs = weights weights_abs[1] = weights_abs[1].abs() # Now I can start my knn algorithm # First I define a distance which is divided by the weights def euclideanDistance(instance1, instance2, start, stop): distance = 0 for x in range(start, stop): distance += (1/abs(weights.iloc[x-3,1]))*pow((instance1[x] - instance2[x]), 2) return math.sqrt(distance) # Then I find the neighbors def getNeighbors(trainingSet, testInstance, k): distances = trainingSet ncol = [] for x in range(len(trainingSet)): dist = euclideanDistance(testInstance, trainingSet.iloc[x,:], 3, len(testInstance)) ncol.append(dist) distances['distance'] = ncol distances = distances.sort(['distance']) neighbor = distances.head(k) return neighbor # I test my function with the 314th line of the dataset test = scaled_features.iloc[314,:] # test_response returns the 20 closest houses test_response = getNeighbors(scaled_features, test, 20) #I want to have test_response in its original form (not standardized) test_response[scaled_col] = scaler.inverse_transform(test_response[scaled_col]) print test_response I made some quick test and the part which is the longer to execute is the KNN part. The standardization and the regression part are quite quick. What do you think of my script ? Do you have ideas to increase its performance ? It is definitely too slow.