When I try to run the following code for arrays with more than 10k elements, it takes hours and I don't know how to make it in the most efficient way.

Any ideas?

from scipy.stats import entropy as KL
import numpy as np

def dis(di,dj):
    di = np.asanyarray(di)
    dj = np.asanyarray(dj)
    m =  0.5 * (di+dj)
    kl1 = KL(di,m)
    kl2 = KL(dj,m)
    return 0.5*(kl1+kl2)

def Intra_Cluster_dist(C):
    C = np.asanyarray(C)
    K = float(C.shape[0])
    factor1 = 1.0/float(K)
    total_sum = 0.0
    for cluster in C:
        cluster = np.asanyarray(cluster)
        below1 = float(cluster.shape[0])
        below2 = float(below1 - 1)
        sub_sum = 0.0
        for di in cluster:
            #others = cluster[:] 
            others = cluster[np.logical_not((cluster == np.array(di)).all(axis=1))]
            #for dj in others:
            #    sub_sum = sub_sum +
            sub_sum = sub_sum + np.fromiter((((2*float(dis(di,dj)))/(float(below1)*float(below2))) for dj in others), dtype=float).sum()
    total_sum = total_sum + sub_sum
    return float(factor1 * total_sum)

def Inter_Cluster_dist(C):
    K = float(len(C))
    factor1 = float((1/(K*(K-1))))
    total_sum = 0.0
    for cluster in C:
        sub_sum = 0.0
        other_clusters = C[:]
        below1= float(len(cluster))
        for other in other_clusters:
            below2= float(len(other))
            for di in cluster:
                for dj in other:
                    sub_sum = sub_sum + (float((dis(di, dj)))/float((below1*below2)))
         total_sum = total_sum + sub_sum
    return float(factor1 * total_sum )

def H_score(C):
    return float(Intra_Cluster_dist(C))/float(Inter_Cluster_dist(C))

Link to example data:

The data file is a xlsx. It has three columns: label (cluster label), feature_1 and feature_2.

The process to get C from the file and get the functions working should be something like this:

import pandas as pd
import numpy as np
df = pd.read_excel('example_data.xlsx')
c1 = np.asanyarray(df[df['labels'] == 0].apply(lambda row: ([row['feature_1'], row['feature_2']]), axis=1))
c2 = np.asanyarray(df[df['labels'] == 1].apply(lambda row: ([row['feature_1'], row['feature_2']]), axis=1))
C = [c1,c2]
  • 2
    \$\begingroup\$ Can you give us example data demonstrating the performance problem? (Or link to it, if too large to add to the post?) \$\endgroup\$ Feb 6 '18 at 9:18
  • \$\begingroup\$ It may help speed-wise to incorporate numpy array functionality in place of nested for-loops (or at least comprehensions). \$\endgroup\$
    – user127168
    Feb 7 '18 at 8:55
  • \$\begingroup\$ Could you post the data in a format other than "pickle", please? Sorry to be a nuisance, but the "pickle" format can run arbitrary Python code when unpickling, and so it is a security risk. \$\endgroup\$ Feb 9 '18 at 11:00
  • \$\begingroup\$ Don't worry, I understand. I've updated the link to an excel file with three columns: Labels (cluster label) , Feature 1, Feature 2. \$\endgroup\$
    – kibs
    Feb 9 '18 at 16:08
  • \$\begingroup\$ Mikey, that sounds promising. Could you please provide an example? \$\endgroup\$
    – kibs
    Feb 9 '18 at 16:23

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.