4
\$\begingroup\$

In the absence of feature-complete and easy-to-use one-hot encoders in the Python ecosystem I've made a set of my own. This is intended to be a small library, so I want to make sure it's as clear and well thought out as possible.

I've implemented things from a previous question concerning only the base encoder, but also expanded it to two separate use cases. Also, let me know if this is not a place for such lengthy code and I'll narrow it down.

I would especially like to know if this is publishable code. So any criticism, on functionality, style or anything is greatly appreciated. Make it harsh too.

import numpy as np
import pandas as pd

class ProgrammingError(Exception):
    """
    Error caused by incorrect use or sequence of routines.
    """

class OneHotEncoder:
    """
    Simple one-hot encoder.

    Does not handle unseen categories: will default to the first category.
    Does not invert all-zero arrays: will default to the first category.
    Does not handle NaN data.

    Example:
        >>> oh = OneHotEncoder()
        >>> oh.fit(np.array(['a', 'b', 'c', 'd']))
        >>> oh.transform(np.array(['a', 'c', 'd', 'a']))
        >>> oh.inverse(np.array([[0, 1, 0, 0]]))
    """
    def __init__(self):
        self._categories = None

    @property
    def categories(self) -> np.ndarray:
        if self._categories is None:
            raise ProgrammingError('Encoder not fitted!')
        return self._categories

    @categories.setter
    def categories(self, categories) -> None:
        self._categories = categories

    @property
    def n_categories(self) -> int:
        return len(self.categories)

    def __repr__(self):
        return 'OneHotEncoder with categories:\n' + str(self.categories)

    def fit(self, samples: np.ndarray) -> 'OneHotEncoder':
        """
        Fit the encoder with the unique elements in categories.

        :param samples: np.ndarray
        :return: None
        """
        self.categories = np.unique(samples)
        return self

    def transform(self, samples: np.ndarray) -> np.ndarray:
        """
        Transform samples into their one-hot encoding.

        :param samples: np.ndarray
        :return: encoding
        """
        return self.transform_from_labels(self.transform_to_labels(samples))

    def transform_to_labels(self, samples: np.ndarray) -> np.ndarray:
        """
        Transform samples to labels (numericals).

        :param samples: np.ndarray
        :return: labels
        """
        arr = np.argwhere(self.categories == samples.reshape(-1, 1))
        labels = np.zeros((samples.size,), dtype=int)
        labels[arr[:, 0]] = arr[:, 1]
        return labels.reshape(samples.shape)

    def transform_from_labels(self, labels: np.ndarray) -> np.ndarray:
        """
        Transform labels to one-hot encoding.

        :param labels: np.ndarray
        :return: encoding
        """
        return np.eye(self.n_categories)[labels]

    def inverse_from_labels(self, labels: np.ndarray) -> np.ndarray:
        """
        Invert labels to original categories.

        :param labels: np.ndarray
        :return: categories
        """
        return self.categories[labels]

    @staticmethod
    def inverse_to_labels(encoded: np.ndarray) -> np.ndarray:
        """
        Invert one-hot encoding to label values

        :param encoded: np.ndarray
        :return: labels
        """
        return np.argmax(encoded, axis=-1)

    def inverse(self, encoded: np.ndarray) -> np.ndarray:
        """
        Invert one-hot encoding to original categories.

        :param encoded: np.ndarray
        :return: categories
        """
        return self.inverse_from_labels(self.inverse_to_labels(encoded))


def _mask_assign(shape: tuple, mask: np.ndarray, values: np.ndarray, init: float=np.nan) -> np.ndarray:
    array = np.full(shape, init)
    array[mask] = values
    return array


class NanHotEncoder(OneHotEncoder):
    """
    One-hot encoder that handles NaN values. Uses pd.isnull to find NaNs.

    Does handle NaN data, ignores unseen categories (all zero) and inverts all zero rows.
    Only accepts and returns 1-dimensional data (pd.Series) as samples (categories).

    Example:
        >>> nh = NanHotEncoder()
        >>> nh.fit(np.array(['a', 'b', 'c', 'd']))
        >>> nh.transform(pd.Series([np.nan, 'c', 'd', 'a']))
        >>> nh.inverse(np.array([[0, 0, 0, 0], [0, 0, 1, 0]]))
    """
    def __init__(self):
        super().__init__()

    def __repr__(self):
        return 'Nan' + super().__repr__()[3:]

    def fit(self, samples: np.ndarray) -> 'NanHotEncoder':
        super().fit(samples[~pd.isnull(samples)])
        return self

    def transform_from_labels(self, labels: np.ndarray) -> np.ndarray:
        nans = np.isnan(labels)
        encoded = super().transform_from_labels(labels[~nans].astype(int))
        return _mask_assign(labels.shape + (self.n_categories,), ~nans, encoded, init=0)

    def inverse_to_lables(self, encoded: np.ndarray) -> np.ndarray:
        nans = np.sum(encoded, axis=-1) == 0
        inverted = super().inverse_to_labels(encoded[~nans].astype(int))
        return _mask_assign(encoded.shape[:-1], ~nans, inverted)

    def transform_to_labels(self, samples: pd.Series) -> np.ndarray:
        mask = samples.isnull() | ~samples.isin(self.categories)
        labels = super().transform_to_labels(samples[~mask].values)
        return _mask_assign(samples.values.shape, ~mask.values, labels)

    def inverse_from_labels(self, labels: np.ndarray) -> pd.Series:
        series = pd.Series(labels.ravel())
        inverted = super().inverse_from_labels(series.dropna().values.astype(int))
        series[~series.isnull()] = inverted
        return series

    def transform(self, samples: pd.Series) -> np.ndarray:
        return self.transform_from_labels(self.transform_to_labels(samples))

    def inverse(self, encoded: np.ndarray) -> pd.Series:
        return self.inverse_from_labels(self.inverse_to_labels(encoded))


class CatHotEncoder(OneHotEncoder):
    """
    One-hot encoder that handles NaN values built around Pandas Categorical type and conventions.

    Does handle NaN data, ignores unseen categories (all zero) and inverts all zero rows.
    Only accepts and returns 1-dimensional data (pd.Series) as samples (categories).

    Example:
        >>> s = pd.Series(pd.Categorical([np.nan, 'c', 'd', 'a', 'b', 'c', 'c']))
        >>> ch = CatHotEncoder()
        >>> ch.fit(s)
        >>> ch.transform(s)
        >>> ch.inverse(np.array([[0, 0, 0, 0], [0, 0, 1, 0]]))
    """
    def __init__(self):
        super().__init__()

    def __repr__(self):
        return 'Cat' + super().__repr__()[3:]

    def fit(self, samples: pd.Series) -> 'CatHotEncoder':
        super().fit(samples.cat.categories)
        return self

    def transform_from_labels(self, labels: np.ndarray) -> np.ndarray:
        nans = (labels == -1)
        encoded = super().transform_from_labels(labels[~nans].astype(int))
        return _mask_assign(labels.shape + (self.n_categories,), ~nans, encoded, init=0)

    def inverse_to_lables(self, encoded: np.ndarray) -> np.ndarray:
        nans = np.sum(encoded, axis=-1) == 0
        inverted = super().inverse_to_labels(encoded[~nans].astype(int))
        return _mask_assign(encoded.shape[:-1], ~nans, inverted, init=-1)

    def transform_to_labels(self, samples: pd.Series) -> np.ndarray:
        raise ProgrammingError('Redundant action for pd.Categorical. Use series.cat.codes instead.')

    def inverse_from_labels(self, labels: np.ndarray) -> pd.Series:
        raise ProgrammingError('Redundant action for pd.Categorical. Use pd.Categorical.from_codes instead.')

    def transform(self, samples: pd.Series) -> np.ndarray:
        return self.transform_from_labels(samples.cat.set_categories(self.categories).cat.codes)

    def inverse(self, encoded: np.ndarray) -> pd.Series:
        codes = self.inverse_to_labels(encoded)
        return pd.Series(pd.Categorical.from_codes(codes, self.categories))

To test it out, please find the examples in each classes docstring or this test suite. Tests are also up for judgement!

import unittest


def array_equal(a: np.ndarray, b: np.ndarray) -> np.ndarray:
    return (a == b) | ((a != a) & (b != b))


class TestOneHotEncoder(unittest.TestCase):
    str_categories = np.array(['a', 'b', 'c', 'd'])

    def setUp(self):
        self.oh = OneHotEncoder().fit(self.str_categories)

    def test_fit(self):
        self.assertTrue(np.all(self.str_categories == self.oh.categories))

    def test_transform_to_labels(self):
        samples = np.array([[['a', 'c'], ['b', 'c']], [['d', 'd'], ['a', 'd']]])
        result = np.array([[[0, 2], [1, 2]], [[3, 3], [0, 3]]])
        self.assertTrue(np.all(self.oh.transform_to_labels(samples) == result))

    def test_transform_from_labels(self):
        labels = np.array([[0, 2], [1, 3]])
        result = np.array([[[1, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 0, 0, 1]]])
        self.assertTrue(np.all(self.oh.transform_from_labels(labels) == result))

    def test_inverse_from_labels(self):
        labels = np.array([[[0, 2], [1, 2]], [[3, 3], [0, 3]]])
        result = np.array([[['a', 'c'], ['b', 'c']], [['d', 'd'], ['a', 'd']]])
        self.assertTrue(np.all(self.oh.inverse_from_labels(labels) == result))

    def test_inverse_to_labels(self):
        encoded = np.array([[[1, 0, 0, 0], [0, 0, 1, 0]], [[0, 1, 0, 0], [0, 0, 0, 1]]])
        result = np.array([[0, 2], [1, 3]])
        self.assertTrue(np.all(self.oh.inverse_to_labels(encoded) == result))


class TestNanHotEncoder(unittest.TestCase):
    categories = np.array(['a', 'b', 'c', 'd', np.nan, np.nan], dtype=object)

    def setUp(self):
        self.nh = NanHotEncoder().fit(self.categories)

    def test_fit(self):
        self.assertTrue(np.all(array_equal(self.nh.categories, self.categories[:-2])))

    def test_transform_to_labels(self):
        samples = pd.Series(['a', 'c', np.nan, 'c', 'd', np.nan, 'a', 'd'])
        result = np.array([0, 2, np.nan, 2, 3, np.nan, 0, 3])
        self.assertTrue(np.all(array_equal(self.nh.transform_to_labels(samples), result)))

    def test_transform_from_labels(self):
        labels = np.array([[0, np.nan], [np.nan, 3]])
        result = np.array([[[1, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 1]]])
        self.assertTrue(np.all(array_equal(self.nh.transform_from_labels(labels), result)))

    def test_inverse_from_labels(self):
        labels = np.array([0, 2, np.nan, 2, 3, np.nan, 0, 3])
        result = pd.Series(['a', 'c', np.nan, 'c', 'd', np.nan, 'a', 'd'])
        self.assertTrue(self.nh.inverse_from_labels(labels).equals(result))

    def test_inverse_to_labels(self):
        encoded = np.array([[[1, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 1]]])
        result = np.array([[0, np.nan], [np.nan, 3]])
        self.assertTrue(np.all(array_equal(self.nh.inverse_to_lables(encoded), result)))

    def test_novel_classes(self):
        samples = pd.Series(['a', 'f', np.nan, 'd'])
        result = np.array([[1, 0, 0, 0], [0, 0, 0, 0],  [0, 0, 0, 0], [0, 0, 0, 1]])
        self.assertTrue(np.all(array_equal(self.nh.transform(samples), result)))


class TestCatHotEncoder(unittest.TestCase):
    series = pd.Series(pd.Categorical([np.nan, 'c', 'd', 'a', 'b', 'c', 'c']))

    def setUp(self):
        self.ch = CatHotEncoder().fit(self.series)

    def test_transform_to_labels(self):
        with self.assertRaises(ProgrammingError):
            self.ch.transform_to_labels(self.series)

    def test_transform_from_labels(self):
        labels = np.array([[0, -1], [-1, 3]])
        result = np.array([[[1, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 1]]])
        self.assertTrue(np.all(array_equal(self.ch.transform_from_labels(labels), result)))

    def test_inverse_from_labels(self):
        with self.assertRaises(ProgrammingError):
            self.ch.transform_to_labels(self.series)

    def test_inverse_to_labels(self):
        encoded = np.array([[[1, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 1]]])
        result = np.array([[0, -1], [-1, 3]])
        self.assertTrue(np.all(array_equal(self.ch.inverse_to_lables(encoded), result)))

    def test_novel_classes(self):
        samples = pd.Series(pd.Categorical(['a', 'f', np.nan, 'd']))
        result = np.array([[1, 0, 0, 0], [0, 0, 0, 0],  [0, 0, 0, 0], [0, 0, 0, 1]])
        self.assertTrue(np.all(array_equal(self.ch.transform(samples), result)))


if __name__ == '__main__':
    oh_test = TestOneHotEncoder()
    nh_test = TestNanHotEncoder()
    ch_test = TestCatHotEncoder()
    test = unittest.TestSuite()
    test.addTests([oh_test, nh_test, ch_test])
    res = unittest.TestResult()
    test.run(res)
\$\endgroup\$
0
\$\begingroup\$

Some improvement's I've come up with myself:

  • Change __repr__ of child classes to not use the parent, slicing the string seems a bit confusing.
  • Return pd.DataFrame from child classes with categories as headers for easy use afterwards.
  • Actually check for the one-dimensionality in transform_from_labels of child classes that was required in docstring but now also enforced by returning a DataFrame.
  • Change test suite accordingly, namely get .values of the DF for tests and pass 1D data.

I may be blind to other mistakes so I still very much welcome other answers!

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.