The following code works, but the filter operation is bottlenecking my analysis. Do you have an idea how I can make the filter operation faster? Do you have other comments about the codestyle?
import sys import pandas as pd import numpy as np from numpy.random import randint, rand, choice, permutation ID = [value for sublist in ((value for _ in range(length)) for value, length in enumerate(randint(1, 10, 70000))) for value in sublist] data = pd.DataFrame(rand(len(ID), 3), columns=['A', 'B', 'C']) data['ID'] = ID data['dt'] = randint(0, sys.maxsize, len(data)).astype('M8[ns]') data.loc[choice([True, False], len(data), [0.05, 0.95]), 'dt'] = None data = data.apply(permutation) # Until here the data was only prepared to be similar to my actual data. # The following command should be optimized. data.groupby('ID').filter(lambda x: not x['dt'].isnull().any())