As you might know, Python's
tarfile does not have the ability to append to the files that compressed with e.g.
bz2. I tried to implement this functionality which works fine but operates slowly. The following function accepts a string or bytes object and appends it as a file to an existing tarfile. I'm not sure if this code has the best performance, and it may have issues. I also tried writing files to memory instead of to a temporary directory, but this didn't impact performance.
import os import tarfile import tempfile import time from pathlib import Path def append_tar_file(buffer, file_name, output_path, replace=True): """ append a buffer to an existing tar file """ # extract files # check for existing file and overwrite if need to # compress files if not os.path.isfile(output_path): return buffer = buffer.encode("utf-8") if isinstance(buffer, str) else buffer with tempfile.TemporaryDirectory() as tempdir: tempdirp = Path(tempdir) with tarfile.open(output_path, "r:bz2") as tar: try: tar.extractall(os.path.abspath(tempdirp)) except Exception as err: #tar file is empty print(err) buffer_path = os.path.join(tempdir, os.path.basename(file_name)) if replace or (buffer_path not in list(os.path.abspath(f) for f in tempdirp.iterdir())): with open(buffer_path, "wb") as f: f.write(buffer) with tarfile.open(output_path, "w:bz2") as tar: for file in tempdirp.iterdir(): try: tar.add(file, arcname=os.path.basename(os.path.normpath(file))) except Exception as err: print(err) if __name__ == "__main__": path = "./test.tar.gz" buffer = "Test String" filename = "somefile" for i in range(1, 100): print(time.time()) append_tar_file(buffer, filename+str(i), path)