Iterate through non-empty words from text files

Question

Given a file or directory, create an iterator that returns the non-empty words from the file or from all files recursively in the directory. Only process ".txt" files. Words are sequence of characters separated by whitespace.

class WordIterable:
    def __init__(self, path: str):
        root = Path(path)
        self._walker: Optional[Iterator[Path]] = None
        if root.is_dir():
            self._walker = root.rglob("*.txt")
        elif root.suffix == ".txt":
            self._walker = (p for p in [root])
        self._open_next_file()
        self._read_next_line()

    def __iter__(self) -> Iterator[str]:
        return self

    def __next__(self) -> str:
        next_word = self._next_word()
        if not next_word:
            self._read_next_line()
            next_word = self._next_word()
        if not next_word:
            self._close_file()
            self._open_next_file()
            self._read_next_line()
            next_word = self._next_word()
        return next_word if WordIterable._is_not_blank(next_word) else next(self)

    def _next_word(self) -> Optional[str]:
        return self._line.pop() if self._line else None

    def _read_next_line(self) -> None:
        self._line = self._fp.readline().split()[::-1]

    def _open_next_file(self) -> None:
        if self._walker:
            self._file: Path = next(self._walker, None)
            if self._file:
                self._fp = self._file.open(encoding="utf8")
                return
        raise StopIteration

    def _close_file(self) -> None:
        self._fp.close()

    @staticmethod
    def _is_not_blank(s: str) -> bool:
        return s and s != "\n"

This works but seems like a lot of code. Can we do better?

Edit:

What is a "word" and a "non-empty word"?

Words are sequence of characters separated by whitespace.

The question doesn't say to recursively processes a directory and it's sub-directories, but that's what the code appears to do.

It does now.

The code only does ".txt" files.

Yes.

Answer 1

The code seems overly complicated and complex for a relatively simple task.

from pathlib import Path

def words(file_or_path):
    path = Path(file_or_path)
    
    if path.is_dir():
        paths = path.rglob('*.txt')
    else:
        paths = (path, )
        
    for filepath in paths:
        yield from filepath.read_text().split()

The function can take a a directory name or a file name. For both cases, create an iterable, paths, of the files to be processed. This way, both cases can be handled by the same code.

For each filepath in paths use Path.read_text() to open the file, read it in, close the file, and return the text that was read. str.split() drops leading and trailing whitespace and then splits the string on whitespace. yield from ... yields each word in turn.

If you don't want to read an entire file in at once, replace the yield from ... with something like:

        with filepath.open() as f:
            for line in f:
                yield from line.split()