3
\$\begingroup\$

I'm implementing the Probabilistic Exponentially Weighted Mean for real time prediction of sensor data in pandas but have issues with optimising the pandas notebook for quick iterations.

Is there a more optimal way to completely remove the for loop as it currently runs longer than expected. How can I take advantage of apply() ; or vectorized operations etc. here?

import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import datetime
import matplotlib.pyplot as plt
#plt.style.use('fivethirtyeight')
#%config InlineBackend.figure_format = 'retina'
#%matplotlib inline
from itertools import islice
from math import sqrt
from scipy.stats import norm


ts = pd.date_range(start ='1-1-2019',  
         end ='1-10-2019', freq ='5T') 


np.random.seed(seed=1111)
data = np.random.normal(2.012547, 1.557331,size=len(ts))
df = pd.DataFrame({'timestamp': ts, 'speed': data})
df.speed = df.speed.abs()
df = df.set_index('timestamp')
time_col = 'timestamp'
value_col = 'speed'

#pewna parameters
T = 30      # initialization period (in cycles)
beta = 0.5  # lower values make the algorithm behave more like regular EWMA
a = 0.99    # the maximum value of the EWMA a parameter, used for outliers
z = 3


#the PEWNA Model

# create a DataFrame for the run time variables we'll need to calculate
pewm = pd.DataFrame(index=df.index, columns=['Mean', 'Var', 'Std'], dtype=float)
pewm.iloc[0] = [df.iloc[0][value_col], 0, 0]

t = 0

for _, row in islice(df.iterrows(), 1, None):
    diff = row[value_col] - pewm.iloc[t].Mean # difference from moving average
    p = norm.pdf(diff / pewm.iloc[t].Std) if pewm.iloc[t].Std != 0 else 0 # Prob of observing diff
    a_t = a * (1 - beta * p) if t > T else 1 - 1/(t+1) # weight to give to this point
    incr = (1 - a_t) * diff

    # Update Mean, Var, Std
    pewm.iloc[t+1].Mean = pewm.iloc[t].Mean + incr
    pewm.iloc[t+1].Var = a_t * (pewm.iloc[t].Var + diff * incr)
    pewm.iloc[t+1].Std = sqrt(pewm.iloc[t+1].Var)
    t += 1

\$\endgroup\$
  • 1
    \$\begingroup\$ @Graipher Ive added dummy data to show what the df looks like. \$\endgroup\$ – Matimba Jun 10 at 9:24

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.