# -*- coding: utf-8 -*-
import csv
import random
import math
import operator
import numpy as np
import pandas as pd, os
from sklearn.preprocessing import OneHotEncoder
from sklearn.feature_extraction import DictVectorizer
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from unidecode import unidecode
import sys
import cProfile
import re
data_pd = pd.read_csv('data.csv')
# I delete rows with NA's (to change later)
data = data.dropna()
# The function I use to deal with categorical data
def one_hot_dataframe(data, cols, replace=False):
vec = DictVectorizer()
mkdict = lambda row: dict((col, row[col]) for col in cols)
vecData = pd.DataFrame(vec.fit_transform(data[cols].apply(mkdict, axis=1)).toarray())
vecData.columns = vec.get_feature_names()
vecData.index = data.index
if replace is True:
data = data.drop(cols, axis=1)
data = data.join(vecData)
return (data, vecData, vec)
# New data set with numerical data only
data_encode, _, _= one_hot_dataframe(data, ['Type', 'Bord_de_mer', 'Bord_de_plan_deau', 'Campagne', 'Centre_ville', 'Complexe_de_vacances', 'Lac', 'Montagne', 'Plage', 'Riviere', 'Village', 'Ville', 'Acces_haut_debit', 'Acces_internet', 'Climatisation', 'Linge_de_maison_fourni', 'Serviettes_de_bain', 'Wifi', 'Terrasse', 'Veranda_Loggia', 'Balcon', 'Jardin', 'location_non_fumeur', 'fumeurs_acceptes', 'animaux_autorises', 'animaux_non_admis', 'acces_handicape', 'Piscine_commune', 'Piscine_privee', 'Piscine_dinterieur', 'Piscine_chauffee', 'Bain_a_remous', 'Sauna'], replace=True)
#convert string to float
data = data.convert_objects(convert_numeric=True)
# deleting an useless column
data = data_encode.drop('SizeIn',1)
# I create a column for the index
data.insert(0, 'index', range(0,len(data)))
# scaled_col the columns to standardize
scaled_col = ['Capacity', 'BedRooms', 'Latitude', 'Longitude', 'Size', 'Acces_haut_debit=No', 'Acces_haut_debit=Yes', 'Acces_internet=No', 'Acces_internet=Yes', 'Bain_a_remous=No', 'Bain_a_remous=Yes', 'Balcon=No', 'Balcon=Yes', 'Bord_de_mer=No', 'Bord_de_plan_deau=No', 'Bord_de_plan_deau=Yes', 'Campagne=No', 'Campagne=Yes', 'Centre_ville=No', 'Centre_ville=Yes', 'Climatisation=No', 'Climatisation=Yes', 'Complexe_de_vacances=No', 'Complexe_de_vacances=Yes', 'Jardin=No', 'Jardin=Yes', 'Lac=No', 'Lac=Yes', 'Linge_de_maison_fourni=No', 'Linge_de_maison_fourni=Yes', 'Montagne=No', 'Montagne=Yes', 'Piscine_chauffee=No', 'Piscine_chauffee=Yes', 'Piscine_commune=No', 'Piscine_commune=Yes', 'Piscine_dinterieur=No', 'Piscine_privee=No', 'Piscine_privee=Yes', 'Plage=No', 'Plage=Yes', 'Riviere=No', 'Riviere=Yes', 'Sauna=No', 'Sauna=Yes', 'Serviettes_de_bain=No', 'Serviettes_de_bain=Yes', 'Terrasse=No', 'Terrasse=Yes', 'Type=Appartement', 'Type=Chalet', "Type=Chambre d'h\xc3\xb4tes", 'Type=G\xc3\xaete', 'Type=Maison', 'Type=Studio', 'Type=Villa', 'Veranda_Loggia=No', 'Veranda_Loggia=Yes', 'Village=No', 'Village=Yes', 'Ville=No', 'Ville=Yes', 'Wifi=No', 'Wifi=Yes', 'acces_handicape=No', 'acces_handicape=Yes', 'animaux_autorises=No', 'animaux_autorises=Yes', 'animaux_non_admis=No', 'animaux_non_admis=Yes', 'fumeurs_acceptes=No', 'fumeurs_acceptes=Yes', 'location_non_fumeur=No', 'location_non_fumeur=Yes']
# standardization of the dataset
scaled_features = data.copy()
features = scaled_features[scaled_col]
scaler = StandardScaler().fit(features.values)
features = scaler.transform(features.values)
scaled_features[scaled_col] = features
# That's the part where I find weights for the knn algorithm
# I'm using a simple regression and I keep the parameters
#create a new dataframe with the concerned features
weightframe = scaled_features
weightframe = weightframe.drop('DetailUrl', axis = 1)
weightframe = weightframe.drop('index', axis = 1)
X = weightframe.drop('Moyenne', axis = 1)
y = weightframe['Moyenne']
lm = LinearRegression(fit_intercept=False)
lm.fit(X,y)
# df with the features and their weights
weights = pd.DataFrame(zip(X.columns, lm.coef_))
# df with positive values
weights_abs = weights
weights_abs[1] = weights_abs[1].abs()
# Now I can start my knn algorithm
# First I define a distance which is divided by the weights
def euclideanDistance(instance1, instance2, start, stop):
distance = 0
for x in range(start, stop):
distance += (1/abs(weights.iloc[x-3,1]))*pow((instance1[x] - instance2[x]), 2)
return math.sqrt(distance)
# Then I find the neighbors
def getNeighbors(trainingSet, testInstance, k):
distances = trainingSet
ncol = []
for x in range(len(trainingSet)):
dist = euclideanDistance(testInstance, trainingSet.iloc[x,:], 3, len(testInstance))
ncol.append(dist)
distances['distance'] = ncol
distances = distances.sort(['distance'])
neighbor = distances.head(k)
return neighbor
# I test my function with the 314th line of the dataset
test = scaled_features.iloc[314,:]
# test_response returns the 20 closest houses
test_response = getNeighbors(scaled_features, test, 20)
#I want to have test_response in its original form (not standardized)
test_response[scaled_col] = scaler.inverse_transform(test_response[scaled_col])
print test_response
My dataset looks like this :
Moyenne DetailUrl Capacity \
0 483.0 http://www.abritel.fr/location-vacances/p1325324 6
1 790.0 http://www.abritel.fr/location-vacances/p1390219 4
2 490.0 http://www.abritel.fr/location-vacances/p1383713 2
3 535.0 http://www.abritel.fr/location-vacances/p1358629 3
4 545.0 http://www.abritel.fr/location-vacances/p2227515a 7
BedRooms Type Latitude Longitude Size SizeIn Bord_de_mer \
0 2 Gîte 47.879710 7.303924 75.0 m² No
1 2 Appartement 47.592455 7.579941 80.0 m² No
2 Studio Studio 48.593275 7.769792 27.0 m² No
3 1 Appartement 48.075133 7.358069 33.0 m² No
4 2 Appartement 48.204212 7.570381 95.0 m² No
Bord_de_plan_deau Campagne Centre_ville Complexe_de_vacances Lac Montagne \
0 No Yes No No No No
1 Yes No Yes No No No
2 No No No No No No
3 No No Yes No No No
4 No Yes No No No No
Plage Riviere Village Ville Acces_haut_debit Acces_internet Climatisation \
0 No No Yes No No Yes No
1 No Yes No Yes No Yes No
2 No No No No No Yes No
3 No No Yes Yes No Yes No
4 No No Yes No No Yes No
Linge_de_maison_fourni Serviettes_de_bain Wifi Terrasse Veranda_Loggia \
0 No No No Yes No
1 Yes Yes No Yes No
2 Yes Yes No Yes No
3 Yes Yes No No No
4 No No No No Yes
Balcon Jardin location_non_fumeur fumeurs_acceptes animaux_autorises \
0 No Yes Yes No Yes
1 Yes Yes Yes No No
2 No No Yes No No
3 No No Yes No No
4 No Yes Yes No No
animaux_non_admis acces_handicape Piscine_commune Piscine_privee \
0 No Yes No No
1 Yes Yes No No
2 Yes Yes No No
3 Yes Yes No No
4 Yes Yes No No
Piscine_dinterieur Piscine_chauffee Bain_a_remous Sauna
0 No No No No
1 No No No No
2 No No No No
3 No No No No
4 No No No No
