Should I put python3 argparse filetype objects in a contextlib stack?

Question

I just read the Lib/argparse.py code (class FileType) on http://hg.python.org/cpython/file/default/Lib/argparse.py. File objects are opened without the with statement. For safe file opening/closing, should I instead encapsulate my argparse.ArgumentParser() in a contextlib stack like below? I feel very stupid for asking a question like this, but I am very much in doubt about the proper procedure.

import argparse
import contextlib
import gzip
import os
import fileinput
import sys


def main():

    with contextlib.ExitStack() as stack:
        input = argparse(stack)
        process_arguments(input)


def argparse(stack):

    parser = argparse.ArgumentParser()
    parser.add_argument('--input', default=[sys.stdin], nargs='+')
    d_args = vars(parser.parse_args())
    if d_args['input'] != [sys.stdin]:
        input = stack.enter_context(fileinput.FileInput(
            files=d_args['input'], openhook=hook_compressed_text))
    else:
        input = stack.enter_context(sys.stdin)

    return input


def hook_compressed_text(filename, mode):

    ext = os.path.splitext(filename)[1]
    if ext == '.gz':
        f = gzip.open(filename, mode + 't')
    else:
        f = open(filename, mode)

    return f

Here the class FileType from Lib/argparse.py:

class FileType(object):
    """Factory for creating file object types

    Instances of FileType are typically passed as type= arguments to the
    ArgumentParser add_argument() method.

    Keyword Arguments:
        - mode -- A string indicating how the file is to be opened. Accepts the
            same values as the builtin open() function.
        - bufsize -- The file's desired buffer size. Accepts the same values as
            the builtin open() function.
        - encoding -- The file's encoding. Accepts the same values as the
            builtin open() function.
        - errors -- A string indicating how encoding and decoding errors are to
            be handled. Accepts the same value as the builtin open() function.
    """

    def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None):
        self._mode = mode
        self._bufsize = bufsize
        self._encoding = encoding
        self._errors = errors

    def __call__(self, string):
        # the special argument "-" means sys.std{in,out}
        if string == '-':
            if 'r' in self._mode:
                return _sys.stdin
            elif 'w' in self._mode:
                return _sys.stdout
            else:
                msg = _('argument "-" with mode %r') % self._mode
                raise ValueError(msg)

        # all other arguments are used as file names
        try:
            return open(string, self._mode, self._bufsize, self._encoding,
                        self._errors)
        except OSError as e:
            message = _("can't open '%s': %s")
            raise ArgumentTypeError(message % (string, e))

Answer 1

I've expanded on your example, and explored some alternatives.

Observations:

• your def argparse shadowed the module; I changed its name.

• using FileType to process the argparse input results in unclosed file warnings. It opens the files, but does not close them. So for multiple files, it is not right tool.

• simpler just runs the FileInput without a context. FileInput handles stdin itself. It does not close stdin when done. It closes other files, even when processing is interrupted.

• main with the context, closes stdin.

• I wrote a process_arguments with a break to test whether files are closed. FileInput properly closes the files, with or without the context.

• FileInput also can read sys.argv, so you don't actually need argparse in this simple case. See simplest.

The script:

import argparse
import contextlib
import fileinput
import sys

files = set()

def filetype():
    parser = argparse.ArgumentParser()
    parser.add_argument('--input', nargs='*',type=argparse.FileType('r'))
    args = parser.parse_args()
    print(args)
    # gives ResourceWarning: unclosed file

def simpler():
    parser = argparse.ArgumentParser()
    parser.add_argument('--input', nargs='*')
    args = parser.parse_args()
    input = fileinput.FileInput(
        files=args.input)
    process_arguments(input)

def simplest():
    # uses sys.argv[1:]
    process_arguments(fileinput.input())    

def main():
    with contextlib.ExitStack() as stack:
        input = myparse(stack)
        process_arguments(input)

def myparse(stack):
    # change name so as to not shadow the module
    parser = argparse.ArgumentParser()
    parser.add_argument('--input', default=[sys.stdin], nargs='+')
    d_args = vars(parser.parse_args())
    print('d_args',d_args)
    if d_args['input'] != [sys.stdin]:
        input = stack.enter_context(fileinput.FileInput(
            files=d_args['input']))
    else:
        input = stack.enter_context(sys.stdin)
    return input

def process_arguments(input):
    try:
        print(input, input._files)
    except AttributeError as e:
        # error if input is not a FileInput
        print(e)
        return
    for l in input:
        print(input.filename(), input.fileno(), input.lineno(), input.filelineno())
        if not input.isstdin():
            files.add(input._file)
        if input.lineno()>15:
            break

if __name__=="__main__":
    # simplest()
    simpler()
    # main()
    if files:
        print([(f.name, f.closed) for f in files])
    print('is stdin closed?', sys.stdin.closed)

This function is a simpler example (without fileinput) of processing a list of files with a proper context. I learned about this use of stdin.fileno() from another argparse bug issue, http://bugs.python.org/issue14156.

def other():
    # example with simple 'with open...'
    parser = argparse.ArgumentParser()
    parser.add_argument('--input', nargs='*', default=[sys.stdin])
    args = parser.parse_args()
    for file in args.input:
        if file is sys.stdin:
            # open(fileno) does not close the underlying file
            file = file.fileno()
        with open(file) as f:
            lines = f.readlines()
            print(f.name, f.fileno(), len(lines))