Append to compressed tar file with performance

Question

As you might know, Python's tarfile does not have the ability to append to the files that compressed with e.g. gz or bz2. I tried to implement this functionality which works fine but operates slowly. The following function accepts a string or bytes object and appends it as a file to an existing tarfile. I'm not sure if this code has the best performance, and it may have issues. I also tried writing files to memory instead of to a temporary directory, but this didn't impact performance.

import os
import tarfile
import tempfile
import time
from pathlib import Path


def append_tar_file(buffer, file_name, output_path, replace=True):
    """
    append a buffer to an existing tar file
    """
    # extract files
    # check for existing file and overwrite if need to
    # compress files
    if not os.path.isfile(output_path):
        return
    buffer = buffer.encode("utf-8") if isinstance(buffer, str) else buffer

    with tempfile.TemporaryDirectory() as tempdir:
        tempdirp = Path(tempdir)
        with tarfile.open(output_path, "r:bz2") as tar:
            try:
                tar.extractall(os.path.abspath(tempdirp))
            except Exception as err:   #tar file is empty
                print(err)
        buffer_path = os.path.join(tempdir, os.path.basename(file_name))
        if replace or (buffer_path not in list(os.path.abspath(f) for f in tempdirp.iterdir())):
            with open(buffer_path, "wb") as f:
                f.write(buffer)

        with tarfile.open(output_path, "w:bz2") as tar:
            for file in tempdirp.iterdir():
                try:
                    tar.add(file, arcname=os.path.basename(os.path.normpath(file)))
                except Exception as err:
                    print(err)

if __name__ == "__main__":
    path = "./test.tar.gz"
    buffer = "Test String"
    filename = "somefile"
    for i in range(1, 100):
        print(time.time())
        append_tar_file(buffer, filename+str(i), path)

Answer 1

Indeed, the tarfile package doesn't support appending to a compressed tar. But I think you can do better than your current attempt. Instead of extracting the content to disk, you could keep it in memory, write and append to a new compressed file, and finally rename the compressed file. That way you will write a single temporary file, instead of many. The reduced disk I/O should improve the performance.

def append_tar_file(buf, file_name, output_path, replace=True):
    """
    append a buf to an existing tar file if not already there, or if replace=True
    """
    if not os.path.isfile(output_path):
        return

    with tempfile.TemporaryDirectory() as tempdir:
        tmp_path = os.path.join(tempdir, 'tmp.tar.bz2')

        with tarfile.open(output_path, "r:bz2") as tar:
            if not replace:
                if file_name in (member.name for member in tar):
                    return

            if isinstance(buf, str):
                buf = buf.encode("utf-8")

            fileobj = BytesIO(buf)
            tarinfo = tarfile.TarInfo(file_name)
            tarinfo.size = len(fileobj.getvalue())

            with tarfile.open(tmp_path, "w:bz2") as tmp:
                for member in tar:
                    if member.name != file_name:
                        tmp.addfile(member, tar.extractfile(member.name))
                tmp.addfile(tarinfo, fileobj)

        os.rename(tmp_path, output_path)

Answer 2

Instead of hardcoding the compression scheme (and possibly compressing a .tar.gz file using BZIP2 as you do), you should try to infer that information. Note that tarfile let you open a compressed file without knowing the compression scheme using tarfile.open(filename, 'r:*') but there is no equivalent for writing the archive.

Since the compression scheme understood by the tarfile module are usual extensions of the files, inspecting the suffixes of output_path should be enough:

def get_compression(filename):
    suffixes = Path(filename).suffixes
    tar, *compression = (s.lstrip('.') for s in suffixes)

    if tar == 'tgz':
        if compression:
            raise RuntimeError('Too much suffixes, cannot infer compression scheme from {}'.format(''.join(suffixes)))
        return 'gz'

    if tar != 'tar':
        raise RuntimeError('Not a tar archive')

    if not compression:
        return ''

    try:
        compression, = compression
    except ValueError:
        raise RuntimeError('Too much compression scheme: {}'.format(', '.join(compression))) from None
    else:
        return compression

Now you can use compression = get_compression(output_path) and then open the tar file using tarfile.open(<name>, 'r:{}'.format(compression)) and open the file for writing using tarfile.open(<name>, 'w:{}'.format(compression)).

Note that I used RuntimeError here but you should probably come up with something less generic.