In the absence of feature-complete and easy-to-use one-hot encoders in the Python ecosystem I've made a set of my own. This is intended to be a small library, so I want to make sure it's as clear and well thought out as possible.
I've implemented things from a previous question concerning only the base encoder, but also expanded it to two separate use cases. Also, let me know if this is not a place for such lengthy code and I'll narrow it down.
I would especially like to know if this is publishable code. So any criticism, on functionality, style or anything is greatly appreciated. Make it harsh too.
import numpy as np
import pandas as pd
class ProgrammingError(Exception):
"""
Error caused by incorrect use or sequence of routines.
"""
class OneHotEncoder:
"""
Simple one-hot encoder.
Does not handle unseen categories: will default to the first category.
Does not invert all-zero arrays: will default to the first category.
Does not handle NaN data.
Example:
>>> oh = OneHotEncoder()
>>> oh.fit(np.array(['a', 'b', 'c', 'd']))
>>> oh.transform(np.array(['a', 'c', 'd', 'a']))
>>> oh.inverse(np.array([[0, 1, 0, 0]]))
"""
def __init__(self):
self._categories = None
@property
def categories(self) -> np.ndarray:
if self._categories is None:
raise ProgrammingError('Encoder not fitted!')
return self._categories
@categories.setter
def categories(self, categories) -> None:
self._categories = categories
@property
def n_categories(self) -> int:
return len(self.categories)
def __repr__(self):
return 'OneHotEncoder with categories:\n' + str(self.categories)
def fit(self, samples: np.ndarray) -> 'OneHotEncoder':
"""
Fit the encoder with the unique elements in categories.
:param samples: np.ndarray
:return: None
"""
self.categories = np.unique(samples)
return self
def transform(self, samples: np.ndarray) -> np.ndarray:
"""
Transform samples into their one-hot encoding.
:param samples: np.ndarray
:return: encoding
"""
return self.transform_from_labels(self.transform_to_labels(samples))
def transform_to_labels(self, samples: np.ndarray) -> np.ndarray:
"""
Transform samples to labels (numericals).
:param samples: np.ndarray
:return: labels
"""
arr = np.argwhere(self.categories == samples.reshape(-1, 1))
labels = np.zeros((samples.size,), dtype=int)
labels[arr[:, 0]] = arr[:, 1]
return labels.reshape(samples.shape)
def transform_from_labels(self, labels: np.ndarray) -> np.ndarray:
"""
Transform labels to one-hot encoding.
:param labels: np.ndarray
:return: encoding
"""
return np.eye(self.n_categories)[labels]
def inverse_from_labels(self, labels: np.ndarray) -> np.ndarray:
"""
Invert labels to original categories.
:param labels: np.ndarray
:return: categories
"""
return self.categories[labels]
@staticmethod
def inverse_to_labels(encoded: np.ndarray) -> np.ndarray:
"""
Invert one-hot encoding to label values
:param encoded: np.ndarray
:return: labels
"""
return np.argmax(encoded, axis=-1)
def inverse(self, encoded: np.ndarray) -> np.ndarray:
"""
Invert one-hot encoding to original categories.
:param encoded: np.ndarray
:return: categories
"""
return self.inverse_from_labels(self.inverse_to_labels(encoded))
def _mask_assign(shape: tuple, mask: np.ndarray, values: np.ndarray, init: float=np.nan) -> np.ndarray:
array = np.full(shape, init)
array[mask] = values
return array
class NanHotEncoder(OneHotEncoder):
"""
One-hot encoder that handles NaN values. Uses pd.isnull to find NaNs.
Does handle NaN data, ignores unseen categories (all zero) and inverts all zero rows.
Only accepts and returns 1-dimensional data (pd.Series) as samples (categories).
Example:
>>> nh = NanHotEncoder()
>>> nh.fit(np.array(['a', 'b', 'c', 'd']))
>>> nh.transform(pd.Series([np.nan, 'c', 'd', 'a']))
>>> nh.inverse(np.array([[0, 0, 0, 0], [0, 0, 1, 0]]))
"""
def __init__(self):
super().__init__()
def __repr__(self):
return 'Nan' + super().__repr__()[3:]
def fit(self, samples: np.ndarray) -> 'NanHotEncoder':
super().fit(samples[~pd.isnull(samples)])
return self
def transform_from_labels(self, labels: np.ndarray) -> np.ndarray:
nans = np.isnan(labels)
encoded = super().transform_from_labels(labels[~nans].astype(int))
return _mask_assign(labels.shape + (self.n_categories,), ~nans, encoded, init=0)
def inverse_to_lables(self, encoded: np.ndarray) -> np.ndarray:
nans = np.sum(encoded, axis=-1) == 0
inverted = super().inverse_to_labels(encoded[~nans].astype(int))
return _mask_assign(encoded.shape[:-1], ~nans, inverted)
def transform_to_labels(self, samples: pd.Series) -> np.ndarray:
mask = samples.isnull() | ~samples.isin(self.categories)
labels = super().transform_to_labels(samples[~mask].values)
return _mask_assign(samples.values.shape, ~mask.values, labels)
def inverse_from_labels(self, labels: np.ndarray) -> pd.Series:
series = pd.Series(labels.ravel())
inverted = super().inverse_from_labels(series.dropna().values.astype(int))
series[~series.isnull()] = inverted
return series
def transform(self, samples: pd.Series) -> np.ndarray:
return self.transform_from_labels(self.transform_to_labels(samples))
def inverse(self, encoded: np.ndarray) -> pd.Series:
return self.inverse_from_labels(self.inverse_to_labels(encoded))
class CatHotEncoder(OneHotEncoder):
"""
One-hot encoder that handles NaN values built around Pandas Categorical type and conventions.
Does handle NaN data, ignores unseen categories (all zero) and inverts all zero rows.
Only accepts and returns 1-dimensional data (pd.Series) as samples (categories).
Example:
>>> s = pd.Series(pd.Categorical([np.nan, 'c', 'd', 'a', 'b', 'c', 'c']))
>>> ch = CatHotEncoder()
>>> ch.fit(s)
>>> ch.transform(s)
>>> ch.inverse(np.array([[0, 0, 0, 0], [0, 0, 1, 0]]))
"""
def __init__(self):
super().__init__()
def __repr__(self):
return 'Cat' + super().__repr__()[3:]
def fit(self, samples: pd.Series) -> 'CatHotEncoder':
super().fit(samples.cat.categories)
return self
def transform_from_labels(self, labels: np.ndarray) -> np.ndarray:
nans = (labels == -1)
encoded = super().transform_from_labels(labels[~nans].astype(int))
return _mask_assign(labels.shape + (self.n_categories,), ~nans, encoded, init=0)
def inverse_to_lables(self, encoded: np.ndarray) -> np.ndarray:
nans = np.sum(encoded, axis=-1) == 0
inverted = super().inverse_to_labels(encoded[~nans].astype(int))
return _mask_assign(encoded.shape[:-1], ~nans, inverted, init=-1)
def transform_to_labels(self, samples: pd.Series) -> np.ndarray:
raise ProgrammingError('Redundant action for pd.Categorical. Use series.cat.codes instead.')
def inverse_from_labels(self, labels: np.ndarray) -> pd.Series:
raise ProgrammingError('Redundant action for pd.Categorical. Use pd.Categorical.from_codes instead.')
def transform(self, samples: pd.Series) -> np.ndarray:
return self.transform_from_labels(samples.cat.set_categories(self.categories).cat.codes)
def inverse(self, encoded: np.ndarray) -> pd.Series:
codes = self.inverse_to_labels(encoded)
return pd.Series(pd.Categorical.from_codes(codes, self.categories))
To test it out, please find the examples in each classes docstring or this test suite. Tests are also up for judgement!
import unittest
def array_equal(a: np.ndarray, b: np.ndarray) -> np.ndarray:
return (a == b) | ((a != a) & (b != b))
class TestOneHotEncoder(unittest.TestCase):
str_categories = np.array(['a', 'b', 'c', 'd'])
def setUp(self):
self.oh = OneHotEncoder().fit(self.str_categories)
def test_fit(self):
self.assertTrue(np.all(self.str_categories == self.oh.categories))
def test_transform_to_labels(self):
samples = np.array([[['a', 'c'], ['b', 'c']], [['d', 'd'], ['a', 'd']]])
result = np.array([[[0, 2], [1, 2]], [[3, 3], [0, 3]]])
self.assertTrue(np.all(self.oh.transform_to_labels(samples) == result))
def test_transform_from_labels(self):
labels = np.array([[0, 2], [1, 3]])
result = np.array([[[1, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 0, 0, 1]]])
self.assertTrue(np.all(self.oh.transform_from_labels(labels) == result))
def test_inverse_from_labels(self):
labels = np.array([[[0, 2], [1, 2]], [[3, 3], [0, 3]]])
result = np.array([[['a', 'c'], ['b', 'c']], [['d', 'd'], ['a', 'd']]])
self.assertTrue(np.all(self.oh.inverse_from_labels(labels) == result))
def test_inverse_to_labels(self):
encoded = np.array([[[1, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 0, 0, 1]]])
result = np.array([[0, 2], [1, 3]])
self.assertTrue(np.all(self.oh.inverse_to_labels(encoded) == result))
class TestNanHotEncoder(unittest.TestCase):
categories = np.array(['a', 'b', 'c', 'd', np.nan, np.nan], dtype=object)
def setUp(self):
self.nh = NanHotEncoder().fit(self.categories)
def test_fit(self):
self.assertTrue(np.all(array_equal(self.nh.categories, self.categories[:-2])))
def test_transform_to_labels(self):
samples = pd.Series(['a', 'c', np.nan, 'c', 'd', np.nan, 'a', 'd'])
result = np.array([0, 2, np.nan, 2, 3, np.nan, 0, 3])
self.assertTrue(np.all(array_equal(self.nh.transform_to_labels(samples), result)))
def test_transform_from_labels(self):
labels = np.array([[0, np.nan], [np.nan, 3]])
result = np.array([[[1, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 1]]])
self.assertTrue(np.all(array_equal(self.nh.transform_from_labels(labels), result)))
def test_inverse_from_labels(self):
labels = np.array([0, 2, np.nan, 2, 3, np.nan, 0, 3])
result = pd.Series(['a', 'c', np.nan, 'c', 'd', np.nan, 'a', 'd'])
self.assertTrue(self.nh.inverse_from_labels(labels).equals(result))
def test_inverse_to_labels(self):
encoded = np.array([[[1, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 1]]])
result = np.array([[0, np.nan], [np.nan, 3]])
self.assertTrue(np.all(array_equal(self.nh.inverse_to_lables(encoded), result)))
def test_novel_classes(self):
samples = pd.Series(['a', 'f', np.nan, 'd'])
result = np.array([[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 1]])
self.assertTrue(np.all(array_equal(self.nh.transform(samples), result)))
class TestCatHotEncoder(unittest.TestCase):
series = pd.Series(pd.Categorical([np.nan, 'c', 'd', 'a', 'b', 'c', 'c']))
def setUp(self):
self.ch = CatHotEncoder().fit(self.series)
def test_transform_to_labels(self):
with self.assertRaises(ProgrammingError):
self.ch.transform_to_labels(self.series)
def test_transform_from_labels(self):
labels = np.array([[0, -1], [-1, 3]])
result = np.array([[[1, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 1]]])
self.assertTrue(np.all(array_equal(self.ch.transform_from_labels(labels), result)))
def test_inverse_from_labels(self):
with self.assertRaises(ProgrammingError):
self.ch.transform_to_labels(self.series)
def test_inverse_to_labels(self):
encoded = np.array([[[1, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 1]]])
result = np.array([[0, -1], [-1, 3]])
self.assertTrue(np.all(array_equal(self.ch.inverse_to_lables(encoded), result)))
def test_novel_classes(self):
samples = pd.Series(pd.Categorical(['a', 'f', np.nan, 'd']))
result = np.array([[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 1]])
self.assertTrue(np.all(array_equal(self.ch.transform(samples), result)))
if __name__ == '__main__':
oh_test = TestOneHotEncoder()
nh_test = TestNanHotEncoder()
ch_test = TestCatHotEncoder()
test = unittest.TestSuite()
test.addTests([oh_test, nh_test, ch_test])
res = unittest.TestResult()
test.run(res)
