Tweeted twitter.com/StackCodeReview/status/967021255340822529

occurred Feb 23, 2018 at 13:00

Update code, so that it is fully working.

Source Link

edited Feb 22, 2018 at 10:12

43.5k
7
76
155

from __future__ import generator_stop
import itertools


def _takewhile(datapredicate, valueiterator, has_data):
    """
    Return successive entries from an iterable as long as the 
    predicate evaluates to true for each entry.

    has_data outputs if the iterator has been consumed in the process.
    """
    for item in dataiterator:
        if predicate(item != value):
            yield item
        else:
            break
    else:
        has_data[0] = False


def isplit(dataiterator, value):
    data"""Return a lazy generator of items in an iterator, seperating by value."""
    iterator = iter(dataiterator)
    has_data = [True]
    while has_data[0]:
        yield _takewhile(datavalue.__ne__, valueiterator, has_data)


def split(dataiterator, value):
    data"""Return a semi-lazy generator of items in an iterator, seperating by value."""
    iterator = iter(dataiterator)
    try:
  has_data = [True]
    while True:
            carry = []
            d = itertools.takewhile_takewhile(value.__ne__, dataiterator, has_data)
        try:
            first = next(d)
        except StopIteration:
            if not has_data[0]:
                break
            yield itertools.chainiter([first],[])
 d, carry)      else:
            carryyield itertools.extendchain([first], d, carry)
    except StopIteration:
        passcarry.extend(d)

print('isplit')
print([list(i) for i in isplit('abc def ghi', ' ')])
print([list(i) for i in isplit(' abc def ghi', ' ')])
s = isplit('abc def ghi', ' ')
print(list(itertools.zip_longest(*itertools.islice(s, 4))))

print('\nsplit')
print([list(i) for i in split('abc def ghi', ' ')])
print([list(i) for i in split(' abc def ghi', ' ')])
s = split('abc def ghi', ' ')
print(list(itertools.zip_longest(*itertools.islice(s, 4))))

isplit
[['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[[], ['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[('a', 'b', 'c', None), ('d', 'e', 'f', None), (None, 'g', 'h', None), (None, 'i', None, None)]

split
[['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[[], ['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[('a', 'd', 'g'), ('b', 'e', 'h'), ('c', 'f', 'i')]

from __future__ import generator_stop
import itertools


def _takewhile(data, value, has_data):
    for item in data:
        if item != value:
            yield item
        else:
            break
    else:
        has_data[0] = False


def isplit(data, value):
    data = iter(data)
    has_data = [True]
    while has_data[0]:
        yield _takewhile(data, value, has_data)


def split(data, value):
    data = iter(data)
    try:
        while True:
            carry = []
            d = itertools.takewhile(value.__ne__, data)
            first = next(d)
            yield itertools.chain([first], d, carry)
            carry.extend(d)
    except StopIteration:
        pass

print('isplit')
print([list(i) for i in isplit('abc def ghi', ' ')])
s = isplit('abc def ghi', ' ')
print(list(itertools.zip_longest(*itertools.islice(s, 4))))

print('\nsplit')
print([list(i) for i in split('abc def ghi', ' ')])
s = split('abc def ghi', ' ')
print(list(itertools.zip_longest(*itertools.islice(s, 4))))

isplit
[['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[('a', 'b', 'c', None), ('d', 'e', 'f', None), (None, 'g', 'h', None), (None, 'i', None, None)]

split
[['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[('a', 'd', 'g'), ('b', 'e', 'h'), ('c', 'f', 'i')]

from __future__ import generator_stop
import itertools


def _takewhile(predicate, iterator, has_data):
    """
    Return successive entries from an iterable as long as the 
    predicate evaluates to true for each entry.

    has_data outputs if the iterator has been consumed in the process.
    """
    for item in iterator:
        if predicate(item):
            yield item
        else:
            break
    else:
        has_data[0] = False


def isplit(iterator, value):
    """Return a lazy generator of items in an iterator, seperating by value."""
    iterator = iter(iterator)
    has_data = [True]
    while has_data[0]:
        yield _takewhile(value.__ne__, iterator, has_data)


def split(iterator, value):
    """Return a semi-lazy generator of items in an iterator, seperating by value."""
    iterator = iter(iterator)
    has_data = [True]
    while True:
        carry = []
        d = _takewhile(value.__ne__, iterator, has_data)
        try:
            first = next(d)
        except StopIteration:
            if not has_data[0]:
                break
            yield iter([])
        else:
            yield itertools.chain([first], d, carry)
            carry.extend(d)

print('isplit')
print([list(i) for i in isplit('abc def ghi', ' ')])
print([list(i) for i in isplit(' abc def ghi', ' ')])
s = isplit('abc def ghi', ' ')
print(list(itertools.zip_longest(*itertools.islice(s, 4))))

print('\nsplit')
print([list(i) for i in split('abc def ghi', ' ')])
print([list(i) for i in split(' abc def ghi', ' ')])
s = split('abc def ghi', ' ')
print(list(itertools.zip_longest(*itertools.islice(s, 4))))

isplit
[['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[[], ['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[('a', 'b', 'c', None), ('d', 'e', 'f', None), (None, 'g', 'h', None), (None, 'i', None, None)]

split
[['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[[], ['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[('a', 'd', 'g'), ('b', 'e', 'h'), ('c', 'f', 'i')]

edited tags

Link

edited Feb 21, 2018 at 19:33

200_success

144.2k
22
188
473

Source Link

asked Feb 21, 2018 at 17:03

Peilonrayz ♦

43.5k
7
76
155

Lazy split and semi-lazy split

Sometimes I need to be able to split data into chunks, and so something like str.split would be helpful. This comes with two downsides:

Input has to be strings
You consume all input when generating the output.

I have a couple of requirements:

It needs to work with any iterable / iterator. Where the items have the != comparator.
I don't want to consume the chunk of data when returning it.
Rather than returning a tuple, I need to return a generator.

And so this left me with two ways to implement the code. A fully lazy version isplit. And one that is semi-lazy where it consumes some of the generator, when moving to the next chunk, without fully consuming it.

And so I created:

from __future__ import generator_stop
import itertools


def _takewhile(data, value, has_data):
    for item in data:
        if item != value:
            yield item
        else:
            break
    else:
        has_data[0] = False


def isplit(data, value):
    data = iter(data)
    has_data = [True]
    while has_data[0]:
        yield _takewhile(data, value, has_data)


def split(data, value):
    data = iter(data)
    try:
        while True:
            carry = []
            d = itertools.takewhile(value.__ne__, data)
            first = next(d)
            yield itertools.chain([first], d, carry)
            carry.extend(d)
    except StopIteration:
        pass

An example of these working are below. There is an edge case with isplit, which is as far as I know inherent from the code being fully lazy. This is shown below too.

print('isplit')
print([list(i) for i in isplit('abc def ghi', ' ')])
s = isplit('abc def ghi', ' ')
print(list(itertools.zip_longest(*itertools.islice(s, 4))))

print('\nsplit')
print([list(i) for i in split('abc def ghi', ' ')])
s = split('abc def ghi', ' ')
print(list(itertools.zip_longest(*itertools.islice(s, 4))))

Which outputs:

isplit
[['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[('a', 'b', 'c', None), ('d', 'e', 'f', None), (None, 'g', 'h', None), (None, 'i', None, None)]

split
[['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']]
[('a', 'd', 'g'), ('b', 'e', 'h'), ('c', 'f', 'i')]

python python-3.x iterator

Stack Exchange Network

Return to Question

Lazy split and semi-lazy split