import functools
from typing import Iterable, Dict
import pandas as pd
import numpy as np
DATA = ...
INDEX = ['89234', '89467', '89519', '89680', '90104']
#COLUMN = np.repeat(['area', 'MPS', 'azimuth'], 9)
# added MULTI_COLUMN per request
DATE_RANGE = pd.to_datetime(range(9), unit='m', origin=pd.Timestamp('1960-01-01'))
MULTI_COLUMN = pd.MultiIndex.from_tuples((
(date, param) for param in ('area', 'MPS', 'azimuth')
for date in DATE_RANGE), names=['validTime', 'parameters'])
def normalize(
frame_cache: pd.DataFrame, thresh) -> pd.DataFrame:
"""normalize values in the dataframe"""
frame_cache[['area', 'MPS', 'azimuth']] = np.block(
list(outliers2nan(frame_cache, thresh=thresh)))
return frame_cache
def outliers2nan(
frame_cache: pd.DataFrame, thresh: Dict[str, float]) -> Iterable[np.ndarray]:
"""
where a parameter is greater than the minimum mean threshold set the value to NaN
"""
for param, frame in frame_cache.groupby('parameters', axis=1, sort=False):
mean = mean_stack(frame, kind=param)
match param:
case 'azimuth':
deg_diff = (
np.rad2deg(frame_cache['azimuth']) - np.rad2deg(mean))
yield frame.where(abs((deg_diff + 180) % 360 - 180) < thresh[param])
case _:
yield frame.where(abs(frame-mean) < mean*thresh[param])
def mean_stack(
frame: pd.DataFrame, axis=1, kind=None) -> np.ndarray:
"""DataFrame.mean(1) stacked as a np.ndarray"""
match kind:
case 'azimuth':
return (
functools.reduce(_mean_azimuth_reduction, frame.to_numpy().T)[:, np.newaxis])
case _:
return (
frame.mean(axis=axis).to_numpy()[:, np.newaxis])
def _mean_azimuth_reduction(col1: np.ndarray, col2: np.ndarray) -> np.ndarray:
np.nan_to_num(x=col1, nan=col2, copy=False)
mean_rads = (
np.arctan2(np.sin(col1)+np.sin(col2), np.cos(col1)+np.cos(col2)))
return mean_rads
def get_means(df: pd.DataFrame) -> pd.DataFrame:
"""start subroutine"""
return pd.DataFrame({
param: mean_stack(frame, kind=param).squeeze()
for param, frame in df.groupby('parameters', axis=1, sort=False)
}, index=INDEX)
def start():
"""construct df and display means"""
bad_df = pd.DataFrame(
DATA, index=INDEX, columns=MULTI_COLUMN).rename_axis(['validTime', 'parameters'], axis=1)
bad_means = get_means(bad_df.droplevel('validTime', axis=1))
good_df = normalize(
bad_df.droplevel('validTime', axis=1).copy(), thresh={'area': .65, 'MPS': .65, 'azimuth': 60.0})
good_means = get_means(good_df)
good_df.columns = MULTI_COLUMN
print(f"""
BAD
{bad_df.stack()}
GOOD
{good_df.stack()}
BAD
{bad_means}
GOOD
{good_means}
DIF
{abs(bad_means-good_means)}
""")
if __name__ == '__main__':
start()
import functools
from typing import Iterable, Dict
import pandas as pd
import numpy as np
DATA = ...
INDEX = ['89234', '89467', '89519', '89680', '90104']
#COLUMN = np.repeat(['area', 'MPS', 'azimuth'], 9)
# added MULTI_COLUMN per request
DATE_RANGE = pd.to_datetime(range(9), unit='m', origin=pd.Timestamp('1960-01-01'))
MULTI_COLUMN = pd.MultiIndex.from_tuples((
(date, param) for param in ('area', 'MPS', 'azimuth')
for date in DATE_RANGE))
def normalize(
frame_cache: pd.DataFrame, thresh) -> pd.DataFrame:
"""normalize values in the dataframe"""
frame_cache[['area', 'MPS', 'azimuth']] = np.block(
list(outliers2nan(frame_cache, thresh=thresh)))
return frame_cache
def outliers2nan(
frame_cache: pd.DataFrame, thresh: Dict[str, float]) -> Iterable[np.ndarray]:
"""
where a parameter is greater than the minimum mean threshold set the value to NaN
"""
for param, frame in frame_cache.groupby('parameters', axis=1, sort=False):
mean = mean_stack(frame, kind=param)
match param:
case 'azimuth':
deg_diff = (
np.rad2deg(frame_cache['azimuth']) - np.rad2deg(mean))
yield frame.where(abs((deg_diff + 180) % 360 - 180) < thresh[param])
case _:
yield frame.where(abs(frame-mean) < mean*thresh[param])
def mean_stack(
frame: pd.DataFrame, axis=1, kind=None) -> np.ndarray:
"""DataFrame.mean(1) stacked as a np.ndarray"""
match kind:
case 'azimuth':
return (
functools.reduce(_mean_azimuth_reduction, frame.to_numpy().T)[:, np.newaxis])
case _:
return (
frame.mean(axis=axis).to_numpy()[:, np.newaxis])
def _mean_azimuth_reduction(col1: np.ndarray, col2: np.ndarray) -> np.ndarray:
np.nan_to_num(x=col1, nan=col2, copy=False)
mean_rads = (
np.arctan2(np.sin(col1)+np.sin(col2), np.cos(col1)+np.cos(col2)))
return mean_rads
def get_means(df: pd.DataFrame) -> pd.DataFrame:
"""start subroutine"""
return pd.DataFrame({
param: mean_stack(frame, kind=param).squeeze()
for param, frame in df.groupby('parameters', axis=1, sort=False)
}, index=INDEX)
def start():
"""construct df and display means"""
bad_df = pd.DataFrame(
DATA, index=INDEX, columns=MULTI_COLUMN).rename_axis(['validTime', 'parameters'], axis=1)
bad_means = get_means(bad_df.droplevel('validTime', axis=1))
good_df = normalize(
bad_df.droplevel('validTime', axis=1).copy(), thresh={'area': .65, 'MPS': .65, 'azimuth': 60.0})
good_means = get_means(good_df)
print(f"""
BAD
{bad_df.stack()}
GOOD
{good_df.stack()}
BAD
{bad_means}
GOOD
{good_means}
DIF
{abs(bad_means-good_means)}
""")
if __name__ == '__main__':
start()
import functools
from typing import Iterable, Dict
import pandas as pd
import numpy as np
DATA = ...
INDEX = ['89234', '89467', '89519', '89680', '90104']
#COLUMN = np.repeat(['area', 'MPS', 'azimuth'], 9)
# added MULTI_COLUMN per request
DATE_RANGE = pd.to_datetime(range(9), unit='m', origin=pd.Timestamp('1960-01-01'))
MULTI_COLUMN = pd.MultiIndex.from_tuples((
(date, param) for param in ('area', 'MPS', 'azimuth')
for date in DATE_RANGE), names=['validTime', 'parameters'])
def normalize(
frame_cache: pd.DataFrame, thresh) -> pd.DataFrame:
"""normalize values in the dataframe"""
frame_cache[['area', 'MPS', 'azimuth']] = np.block(
list(outliers2nan(frame_cache, thresh=thresh)))
return frame_cache
def outliers2nan(
frame_cache: pd.DataFrame, thresh: Dict[str, float]) -> Iterable[np.ndarray]:
"""
where a parameter is greater than the minimum mean threshold set the value to NaN
"""
for param, frame in frame_cache.groupby('parameters', axis=1, sort=False):
mean = mean_stack(frame, kind=param)
match param:
case 'azimuth':
deg_diff = (
np.rad2deg(frame_cache['azimuth']) - np.rad2deg(mean))
yield frame.where(abs((deg_diff + 180) % 360 - 180) < thresh[param])
case _:
yield frame.where(abs(frame-mean) < mean*thresh[param])
def mean_stack(
frame: pd.DataFrame, axis=1, kind=None) -> np.ndarray:
"""DataFrame.mean(1) stacked as a np.ndarray"""
match kind:
case 'azimuth':
return (
functools.reduce(_mean_azimuth_reduction, frame.to_numpy().T)[:, np.newaxis])
case _:
return (
frame.mean(axis=axis).to_numpy()[:, np.newaxis])
def _mean_azimuth_reduction(col1: np.ndarray, col2: np.ndarray) -> np.ndarray:
np.nan_to_num(x=col1, nan=col2, copy=False)
mean_rads = (
np.arctan2(np.sin(col1)+np.sin(col2), np.cos(col1)+np.cos(col2)))
return mean_rads
def get_means(df: pd.DataFrame) -> pd.DataFrame:
"""start subroutine"""
return pd.DataFrame({
param: mean_stack(frame, kind=param).squeeze()
for param, frame in df.groupby('parameters', axis=1, sort=False)
}, index=INDEX)
def start():
"""construct df and display means"""
bad_df = pd.DataFrame(
DATA, index=INDEX, columns=MULTI_COLUMN)
bad_means = get_means(bad_df)
good_df = normalize(
bad_df.droplevel('validTime', axis=1).copy(), thresh={'area': .65, 'MPS': .65, 'azimuth': 60.0})
good_means = get_means(good_df)
good_df.columns = MULTI_COLUMN
print(f"""
BAD
{bad_df.stack()}
GOOD
{good_df.stack()}
BAD
{bad_means}
GOOD
{good_means}
DIF
{abs(bad_means-good_means)}
""")
if __name__ == '__main__':
start()
import functools
from typing import Iterable, Dict
import pandas as pd
import numpy as np
DATA = ...
INDEX = ['89234', '89467', '89519', '89680', '90104']
COLUMN#COLUMN = np.repeat(['area', 'MPS', 'azimuth'], 9)
# added MULTI_COLUMN per request
DATE_RANGE = pd.to_datetime(range(9), unit='m', origin=pd.Timestamp('1960-01-01'))
MULTI_COLUMN = pd.MultiIndex.from_tuples((
(date, param) for param in ('area', 'MPS', 'azimuth')
for date in DATE_RANGE))
def normalize(
frame_cache: pd.DataFrame, thresh) -> pd.DataFrame:
"""normalize values in the dataframe"""
frame_cache[['area', 'MPS', 'azimuth']] = np.block(
list(outliers2nan(frame_cache, thresh=thresh)))
return frame_cache
def outliers2nan(
frame_cache: pd.DataFrame, thresh: Dict[str, float]) -> Iterable[np.ndarray]:
"""
where a parameter is greater than the minimum mean threshold set the value to NaN
"""
for param, frame in frame_cache.groupby('parameters', axis=1, sort=False):
mean = mean_stack(frame, kind=param)
match param:
case 'azimuth':
deg_diff = (
np.rad2deg(frame_cache['azimuth']) - np.rad2deg(mean))
yield frame.where(abs((deg_diff + 180) % 360 - 180) < thresh[param])
case _:
yield frame.where(abs(frame-mean) < mean*thresh[param])
def mean_stack(
frame: pd.DataFrame, axis=1, kind=None) -> np.ndarray:
"""DataFrame.mean(1) stacked as a np.ndarray"""
match kind:
case 'azimuth':
return (
functools.reduce(_mean_azimuth_reduction, frame.to_numpy().T)[:, np.newaxis])
case _:
return (
frame.mean(axis=axis).to_numpy()[:, np.newaxis])
def _mean_azimuth_reduction(col1: np.ndarray, col2: np.ndarray) -> np.ndarray:
np.nan_to_num(x=col1, nan=col2, copy=False)
mean_rads = (
np.arctan2(np.sin(col1)+np.sin(col2), np.cos(col1)+np.cos(col2)))
return mean_rads
def get_means(df: pd.DataFrame) -> pd.DataFrame:
"""start subroutine"""
return pd.DataFrame({
param: mean_stack(frame, kind=param).squeeze()
for param, frame in df.groupby('parameters', axis=1, sort=False)
}, index=INDEX)
def start():
"""construct df and display means"""
bad_df = pd.DataFrame(
DATA, index=INDEX, columns=COLUMNcolumns=MULTI_COLUMN).rename_axis('parameters'['validTime', 'parameters'], axis=1)
bad_means = get_means(bad_df.droplevel('validTime', axis=1))
good_df = normalize(
bad_df.droplevel('validTime', axis=1).copy(), thresh={'area': .65, 'MPS': .65, 'azimuth': 60.0})
good_means = get_means(good_df)
print(f"""
BAD
{bad_df.stack()}
GOOD
{good_df.stack()}
BAD
{bad_means}
GOOD
{good_means}
DIF
{abs(bad_means-good_means)}
""")
if __name__ == '__main__':
start()
BAD
validTime 1960-01-01 00:00:00 1960-01-01 00:01:00 1960-01-01 00:02:00 1960-01-01 00:03:00 1960-01-01 00:04:00 1960-01-01 00:05:00 1960-01-01 00:06:00 1960-01-01 00:07:00 1960-01-01 00:08:00
parameters area area area area area area area area area MPS MPS MPS MPS MPS MPS MPS MPS MPS azimuth azimuth azimuth azimuth azimuth azimuth azimuth azimuth azimuth
89234 0.04575 0.04940 0.05245 0.05440 0.05610 0.05375 0.04770 0.05370 0.05325
89234 MPS 16.812098 27.983008 20.301827 13.421484 17.105572 14.686988 41.180826 13.065487 4.608155
area 0.757195045750 0.345778049400 0.460570052450 0.281990054400 0.720571056100 - 0.000410053750 0.386078047700 0.096300053700 0.246872053250
89467 azimuth 0.03290757195 0.03355345778 0.03950460570 0.03555281990 0.03725720571 -0.03515000410 0.03715386078 0.04135096300 0.03870246872
89467 MPS 14.902912 15.452401 12.548454 10.837832 13.246734 12.636336 19.610936 8.898328 26.237158
area 0.365896032900 0.514060033550 0.060924039500 0.543359035550 0.330022037250 0.129570035150 0.581377037150 0.978887041350 0.214447038700
89519 azimuth 0.15945365896 0.14190514060 0.13460060924 0.14315543359 0.14485330022 0.12930129570 0.14265581377 0.14345978887 0.08945214447
89519 MPS 39.549821 67.055990 14.170464 25.571033 22.224606 28.901344 19.051307 25.612899 122.314415
area 0.525051159450 0.311914141900 0.381809134600 0.266956143150 - 0.487672144850 0.038141129300 - 0.075021142650 0.153714143450 -1 0.102629089450
89680 azimuth 0.00340525051 0.00660311914 0.00850381809 0.00800266956 -0.00655487672 0.00810038141 -0.00325075021 0.01835153714 0 -1.02015102629
89680 MPS 22.364277 20.506558 6.243037 19.345872 10.456834 19.658797 14.307244 9.091404 18.474207
area 0.848569003400 0.400943006600 0.115777008500 0.694402008000 0.467667006550 1 0.153658008100 0.760046003250 0.417047018350 0.499209020150
90104 azimuth NaN NaN 0.00355848569 0.00580400943 0.00580115777 0.00690694402 0.00705467667 1.153658 0.00680760046 0.00520417047 0.499209
90104 MPS NaN NaN 8.061103 13.695219 0.000000 12.818729 6.190861 3.986124 4.403761
area NaN NaN 0.003550 0.005800 0.005800 0.006900 0.007050 0.006800 0.005200
azimuth NaN NaN -0.834234 -0.359909 0.000000 -2.970813 -0.123782 -1.464691 -2.475146
GOOD
parameters area area area1960-01-01 00:00:00 1960-01-01 00:01:00 area 1960-01-01 00:02:00 1960-01-01 area00:03:00 1960-01-01 00:04:00 area1960-01-01 00:05:00 1960-01-01 00:06:00 area 1960-01-01 00:07:00 1960-01-01 area00:08:00
89234 MPS area MPS 16.812098 MPS MPS27.983008 MPS 20.301827 MPS MPS13.421484 MPS 17.105572 MPS MPS14.686988 azimuth azimuth azimuth azimuth azimuth azimuthNaN azimuth azimuth azimuth 13.065487 NaN
89234 area 0.04575 0.04940 0.05245045750 0.05440049400 0.05610052450 0.05375054400 0.04770056100 0.05370053750 0.05325047700 16.812098 27.983008 20.301827 13.421484 17.105572 14 0.686988053700 NaN 13 0.065487053250
azimuth NaN 0.757195 0.345778 0.460570 0.281990 0.720571 -0.000410 0.386078 0.096300 0.246872
89467 MPS 0.03290 0.03355 0.03950 0.03555 0.03725 014.03515902912 0.03715 0.04135 0.03870 14.902912 15.452401 12.548454 10.837832 13.246734 12.636336 19.610936 8.898328 NaN
area 0.365896032900 0.514060033550 0.060924039500 0.543359035550 0.330022037250 0.129570035150 0.581377037150 0.978887041350 0.214447038700
89519 azimuth 0.15945365896 0.14190514060 0.13460060924 0.14315543359 0.14485330022 0.12930129570 0.14265581377 0.14345978887 0.08945214447
89519 MPS 39.549821 NaN NaN 25.571033 22.224606 28.901344 19.051307 25.612899 NaN NaN
area 0.311914159450 0.381809141900 0.266956134600 - 0.487672143150 0.038141144850 - 0.075021129300 0.153714142650 -1 0.102629143450 0.089450
89680 azimuth NaN 0.00340311914 0.00660381809 0.00850266956 -0.00800487672 0.00655038141 -0.00810075021 0.00325153714 NaN NaN-1.102629
89680 MPS 22.364277 20.506558 6.243037 19.345872 10.456834 19.658797 14.307244 9.091404 18.474207
0.848569 area 0.400943003400 0.115777006600 0.694402008500 0.467667008000 1.153658 0.760046006550 0.417047008100 0.499209
90104003250 NaN NaN
azimuth 0.00355848569 0.00580400943 0.00580115777 0.00690694402 0.00705467667 1.153658 0.00680760046 0.00520417047 0.499209
90104 MPS NaN NaN 8.061103 NaN NaN NaN 6.190861 3.986124 4.403761
area NaN NaN 0.003550 0.005800 0.005800 0.006900 0.007050 0.006800 0.005200
azimuth NaN NaN -0.834234 NaN NaN NaN NaN -1.464691 -2.475146
BAD
area MPS azimuth
89234 0.051833 18.796161 0.230576
89467 0.036789 14.930121 0.455432
89519 0.136533 40.494653 -0.524696
89680 0.009211 15.605359 0.552226
90104 0.005871 7.022257 -1.823553
GOOD
area MPS azimuth
89234 0.051833 17.625209 0.230576
89467 0.036789 13.516742 0.455432
89519 0.136533 26.818502 -0.525529
89680 0.006343 15.605359 0.552226
90104 0.005871 5.660462 -1.969919
DIF
area MPS azimuth
89234 0.000000 1.170951 0.000000
89467 0.000000 1.413380 0.000000
89519 0.000000 13.676152 0.000833
89680 0.002868 0.000000 0.000000
90104 0.000000 1.361794 0.146366
import functools
from typing import Iterable, Dict
import pandas as pd
import numpy as np
DATA = ...
INDEX = ['89234', '89467', '89519', '89680', '90104']
COLUMN = np.repeat(['area', 'MPS', 'azimuth'], 9)
def normalize(
frame_cache: pd.DataFrame, thresh) -> pd.DataFrame:
"""normalize values in the dataframe"""
frame_cache[['area', 'MPS', 'azimuth']] = np.block(
list(outliers2nan(frame_cache, thresh=thresh)))
return frame_cache
def outliers2nan(
frame_cache: pd.DataFrame, thresh: Dict[str, float]) -> Iterable[np.ndarray]:
"""
where a parameter is greater than the minimum mean threshold set the value to NaN
"""
for param, frame in frame_cache.groupby('parameters', axis=1, sort=False):
mean = mean_stack(frame, kind=param)
match param:
case 'azimuth':
deg_diff = (
np.rad2deg(frame_cache['azimuth']) - np.rad2deg(mean))
yield frame.where(abs((deg_diff + 180) % 360 - 180) < thresh[param])
case _:
yield frame.where(abs(frame-mean) < mean*thresh[param])
def mean_stack(
frame: pd.DataFrame, axis=1, kind=None) -> np.ndarray:
"""DataFrame.mean(1) stacked as a np.ndarray"""
match kind:
case 'azimuth':
return (
functools.reduce(_mean_azimuth_reduction, frame.to_numpy().T)[:, np.newaxis])
case _:
return (
frame.mean(axis=axis).to_numpy()[:, np.newaxis])
def _mean_azimuth_reduction(col1: np.ndarray, col2: np.ndarray) -> np.ndarray:
np.nan_to_num(x=col1, nan=col2, copy=False)
mean_rads = (
np.arctan2(np.sin(col1)+np.sin(col2), np.cos(col1)+np.cos(col2)))
return mean_rads
def get_means(df: pd.DataFrame) -> pd.DataFrame:
"""start subroutine"""
return pd.DataFrame({
param: mean_stack(frame, kind=param).squeeze()
for param, frame in df.groupby('parameters', axis=1, sort=False)
}, index=INDEX)
def start():
"""construct df and display means"""
bad_df = pd.DataFrame(
DATA, index=INDEX, columns=COLUMN).rename_axis('parameters', axis=1)
bad_means = get_means(bad_df)
good_df = normalize(
bad_df.copy(), thresh={'area': .65, 'MPS': .65, 'azimuth': 60.0})
good_means = get_means(good_df)
print(f"""
BAD
{bad_df}
GOOD
{good_df}
BAD
{bad_means}
GOOD
{good_means}
DIF
{abs(bad_means-good_means)}
""")
if __name__ == '__main__':
start()
BAD
parameters area area area area area area area area area MPS MPS MPS MPS MPS MPS MPS MPS MPS azimuth azimuth azimuth azimuth azimuth azimuth azimuth azimuth azimuth
89234 0.04575 0.04940 0.05245 0.05440 0.05610 0.05375 0.04770 0.05370 0.05325 16.812098 27.983008 20.301827 13.421484 17.105572 14.686988 41.180826 13.065487 4.608155 0.757195 0.345778 0.460570 0.281990 0.720571 -0.000410 0.386078 0.096300 0.246872
89467 0.03290 0.03355 0.03950 0.03555 0.03725 0.03515 0.03715 0.04135 0.03870 14.902912 15.452401 12.548454 10.837832 13.246734 12.636336 19.610936 8.898328 26.237158 0.365896 0.514060 0.060924 0.543359 0.330022 0.129570 0.581377 0.978887 0.214447
89519 0.15945 0.14190 0.13460 0.14315 0.14485 0.12930 0.14265 0.14345 0.08945 39.549821 67.055990 14.170464 25.571033 22.224606 28.901344 19.051307 25.612899 122.314415 0.525051 0.311914 0.381809 0.266956 -0.487672 0.038141 -0.075021 0.153714 -1.102629
89680 0.00340 0.00660 0.00850 0.00800 0.00655 0.00810 0.00325 0.01835 0.02015 22.364277 20.506558 6.243037 19.345872 10.456834 19.658797 14.307244 9.091404 18.474207 0.848569 0.400943 0.115777 0.694402 0.467667 1.153658 0.760046 0.417047 0.499209
90104 NaN NaN 0.00355 0.00580 0.00580 0.00690 0.00705 0.00680 0.00520 NaN NaN 8.061103 13.695219 0.000000 12.818729 6.190861 3.986124 4.403761 NaN NaN -0.834234 -0.359909 0.000000 -2.970813 -0.123782 -1.464691 -2.475146
GOOD
parameters area area area area area area area area area MPS MPS MPS MPS MPS MPS MPS MPS MPS azimuth azimuth azimuth azimuth azimuth azimuth azimuth azimuth azimuth
89234 0.04575 0.04940 0.05245 0.05440 0.05610 0.05375 0.04770 0.05370 0.05325 16.812098 27.983008 20.301827 13.421484 17.105572 14.686988 NaN 13.065487 NaN 0.757195 0.345778 0.460570 0.281990 0.720571 -0.000410 0.386078 0.096300 0.246872
89467 0.03290 0.03355 0.03950 0.03555 0.03725 0.03515 0.03715 0.04135 0.03870 14.902912 15.452401 12.548454 10.837832 13.246734 12.636336 19.610936 8.898328 NaN 0.365896 0.514060 0.060924 0.543359 0.330022 0.129570 0.581377 0.978887 0.214447
89519 0.15945 0.14190 0.13460 0.14315 0.14485 0.12930 0.14265 0.14345 0.08945 39.549821 NaN NaN 25.571033 22.224606 28.901344 19.051307 25.612899 NaN NaN 0.311914 0.381809 0.266956 -0.487672 0.038141 -0.075021 0.153714 -1.102629
89680 0.00340 0.00660 0.00850 0.00800 0.00655 0.00810 0.00325 NaN NaN 22.364277 20.506558 6.243037 19.345872 10.456834 19.658797 14.307244 9.091404 18.474207 0.848569 0.400943 0.115777 0.694402 0.467667 1.153658 0.760046 0.417047 0.499209
90104 NaN NaN 0.00355 0.00580 0.00580 0.00690 0.00705 0.00680 0.00520 NaN NaN 8.061103 NaN NaN NaN 6.190861 3.986124 4.403761 NaN NaN -0.834234 NaN NaN NaN NaN -1.464691 -2.475146
BAD
area MPS azimuth
89234 0.051833 18.796161 0.230576
89467 0.036789 14.930121 0.455432
89519 0.136533 40.494653 -0.524696
89680 0.009211 15.605359 0.552226
90104 0.005871 7.022257 -1.823553
GOOD
area MPS azimuth
89234 0.051833 17.625209 0.230576
89467 0.036789 13.516742 0.455432
89519 0.136533 26.818502 -0.525529
89680 0.006343 15.605359 0.552226
90104 0.005871 5.660462 -1.969919
DIF
area MPS azimuth
89234 0.000000 1.170951 0.000000
89467 0.000000 1.413380 0.000000
89519 0.000000 13.676152 0.000833
89680 0.002868 0.000000 0.000000
90104 0.000000 1.361794 0.146366
import functools
from typing import Iterable, Dict
import pandas as pd
import numpy as np
DATA = ...
INDEX = ['89234', '89467', '89519', '89680', '90104']
#COLUMN = np.repeat(['area', 'MPS', 'azimuth'], 9)
# added MULTI_COLUMN per request
DATE_RANGE = pd.to_datetime(range(9), unit='m', origin=pd.Timestamp('1960-01-01'))
MULTI_COLUMN = pd.MultiIndex.from_tuples((
(date, param) for param in ('area', 'MPS', 'azimuth')
for date in DATE_RANGE))
def normalize(
frame_cache: pd.DataFrame, thresh) -> pd.DataFrame:
"""normalize values in the dataframe"""
frame_cache[['area', 'MPS', 'azimuth']] = np.block(
list(outliers2nan(frame_cache, thresh=thresh)))
return frame_cache
def outliers2nan(
frame_cache: pd.DataFrame, thresh: Dict[str, float]) -> Iterable[np.ndarray]:
"""
where a parameter is greater than the minimum mean threshold set the value to NaN
"""
for param, frame in frame_cache.groupby('parameters', axis=1, sort=False):
mean = mean_stack(frame, kind=param)
match param:
case 'azimuth':
deg_diff = (
np.rad2deg(frame_cache['azimuth']) - np.rad2deg(mean))
yield frame.where(abs((deg_diff + 180) % 360 - 180) < thresh[param])
case _:
yield frame.where(abs(frame-mean) < mean*thresh[param])
def mean_stack(
frame: pd.DataFrame, axis=1, kind=None) -> np.ndarray:
"""DataFrame.mean(1) stacked as a np.ndarray"""
match kind:
case 'azimuth':
return (
functools.reduce(_mean_azimuth_reduction, frame.to_numpy().T)[:, np.newaxis])
case _:
return (
frame.mean(axis=axis).to_numpy()[:, np.newaxis])
def _mean_azimuth_reduction(col1: np.ndarray, col2: np.ndarray) -> np.ndarray:
np.nan_to_num(x=col1, nan=col2, copy=False)
mean_rads = (
np.arctan2(np.sin(col1)+np.sin(col2), np.cos(col1)+np.cos(col2)))
return mean_rads
def get_means(df: pd.DataFrame) -> pd.DataFrame:
"""start subroutine"""
return pd.DataFrame({
param: mean_stack(frame, kind=param).squeeze()
for param, frame in df.groupby('parameters', axis=1, sort=False)
}, index=INDEX)
def start():
"""construct df and display means"""
bad_df = pd.DataFrame(
DATA, index=INDEX, columns=MULTI_COLUMN).rename_axis(['validTime', 'parameters'], axis=1)
bad_means = get_means(bad_df.droplevel('validTime', axis=1))
good_df = normalize(
bad_df.droplevel('validTime', axis=1).copy(), thresh={'area': .65, 'MPS': .65, 'azimuth': 60.0})
good_means = get_means(good_df)
print(f"""
BAD
{bad_df.stack()}
GOOD
{good_df.stack()}
BAD
{bad_means}
GOOD
{good_means}
DIF
{abs(bad_means-good_means)}
""")
if __name__ == '__main__':
start()
BAD
validTime 1960-01-01 00:00:00 1960-01-01 00:01:00 1960-01-01 00:02:00 1960-01-01 00:03:00 1960-01-01 00:04:00 1960-01-01 00:05:00 1960-01-01 00:06:00 1960-01-01 00:07:00 1960-01-01 00:08:00
parameters
89234 MPS 16.812098 27.983008 20.301827 13.421484 17.105572 14.686988 41.180826 13.065487 4.608155
area 0.045750 0.049400 0.052450 0.054400 0.056100 0.053750 0.047700 0.053700 0.053250
azimuth 0.757195 0.345778 0.460570 0.281990 0.720571 -0.000410 0.386078 0.096300 0.246872
89467 MPS 14.902912 15.452401 12.548454 10.837832 13.246734 12.636336 19.610936 8.898328 26.237158
area 0.032900 0.033550 0.039500 0.035550 0.037250 0.035150 0.037150 0.041350 0.038700
azimuth 0.365896 0.514060 0.060924 0.543359 0.330022 0.129570 0.581377 0.978887 0.214447
89519 MPS 39.549821 67.055990 14.170464 25.571033 22.224606 28.901344 19.051307 25.612899 122.314415
area 0.159450 0.141900 0.134600 0.143150 0.144850 0.129300 0.142650 0.143450 0.089450
azimuth 0.525051 0.311914 0.381809 0.266956 -0.487672 0.038141 -0.075021 0.153714 -1.102629
89680 MPS 22.364277 20.506558 6.243037 19.345872 10.456834 19.658797 14.307244 9.091404 18.474207
area 0.003400 0.006600 0.008500 0.008000 0.006550 0.008100 0.003250 0.018350 0.020150
azimuth 0.848569 0.400943 0.115777 0.694402 0.467667 1.153658 0.760046 0.417047 0.499209
90104 MPS NaN NaN 8.061103 13.695219 0.000000 12.818729 6.190861 3.986124 4.403761
area NaN NaN 0.003550 0.005800 0.005800 0.006900 0.007050 0.006800 0.005200
azimuth NaN NaN -0.834234 -0.359909 0.000000 -2.970813 -0.123782 -1.464691 -2.475146
GOOD
1960-01-01 00:00:00 1960-01-01 00:01:00 1960-01-01 00:02:00 1960-01-01 00:03:00 1960-01-01 00:04:00 1960-01-01 00:05:00 1960-01-01 00:06:00 1960-01-01 00:07:00 1960-01-01 00:08:00
89234 MPS 16.812098 27.983008 20.301827 13.421484 17.105572 14.686988 NaN 13.065487 NaN
area 0.045750 0.049400 0.052450 0.054400 0.056100 0.053750 0.047700 0.053700 0.053250
azimuth 0.757195 0.345778 0.460570 0.281990 0.720571 -0.000410 0.386078 0.096300 0.246872
89467 MPS 14.902912 15.452401 12.548454 10.837832 13.246734 12.636336 19.610936 8.898328 NaN
area 0.032900 0.033550 0.039500 0.035550 0.037250 0.035150 0.037150 0.041350 0.038700
azimuth 0.365896 0.514060 0.060924 0.543359 0.330022 0.129570 0.581377 0.978887 0.214447
89519 MPS 39.549821 NaN NaN 25.571033 22.224606 28.901344 19.051307 25.612899 NaN
area 0.159450 0.141900 0.134600 0.143150 0.144850 0.129300 0.142650 0.143450 0.089450
azimuth NaN 0.311914 0.381809 0.266956 -0.487672 0.038141 -0.075021 0.153714 -1.102629
89680 MPS 22.364277 20.506558 6.243037 19.345872 10.456834 19.658797 14.307244 9.091404 18.474207
area 0.003400 0.006600 0.008500 0.008000 0.006550 0.008100 0.003250 NaN NaN
azimuth 0.848569 0.400943 0.115777 0.694402 0.467667 1.153658 0.760046 0.417047 0.499209
90104 MPS NaN NaN 8.061103 NaN NaN NaN 6.190861 3.986124 4.403761
area NaN NaN 0.003550 0.005800 0.005800 0.006900 0.007050 0.006800 0.005200
azimuth NaN NaN -0.834234 NaN NaN NaN NaN -1.464691 -2.475146
BAD
area MPS azimuth
89234 0.051833 18.796161 0.230576
89467 0.036789 14.930121 0.455432
89519 0.136533 40.494653 -0.524696
89680 0.009211 15.605359 0.552226
90104 0.005871 7.022257 -1.823553
GOOD
area MPS azimuth
89234 0.051833 17.625209 0.230576
89467 0.036789 13.516742 0.455432
89519 0.136533 26.818502 -0.525529
89680 0.006343 15.605359 0.552226
90104 0.005871 5.660462 -1.969919
DIF
area MPS azimuth
89234 0.000000 1.170951 0.000000
89467 0.000000 1.413380 0.000000
89519 0.000000 13.676152 0.000833
89680 0.002868 0.000000 0.000000
90104 0.000000 1.361794 0.146366
I am using matchmatch statements so you will need python 3.10
A good example of the use case is on the last MPS column of index 89519 which reported a speed of 122 meters per second. That value gets omitted from the final return value.
I am using match statements so you will need python 3.10
I am using match statements so you will need python 3.10
A good example of the use case is on the last MPS column of index 89519 which reported a speed of 122 meters per second. That value gets omitted from the final return value.
Loading
Loading
lang-py