Foldify - A Python Folder Tree Manager Tool

Question

The goal was to create a tool to help manage folders structures, and allow me to create templates of this folders and store them as a JSON template. The following functionalities have been implemented:

1. Make a copy of a directory tree (folders only).
2. Create a JSON template of the directory tree.
3. Create a directory tree from a JSON template.

There are 3 modules:

1. path_tree.py: path object class to help navigate directory and turn it into a tree object.
2. core.py: main functionalities, handles exceptions
3. foldify.py: CLI interface for executing the tasks

I would appreciate any feedback on improving the code or its organization.

Repo:

[email protected]:gtalarico/foldify.git

https://github.com/gtalarico/foldify

path_tree.py: a PathObject (PathO) helper class to help deal with folder hierarchy relationships (children, iterate up and down, get root, etc)

import os
import json
from collections import OrderedDict


class PathO(object):

    def __init__(self, name, static_fullpath=None, children=[], parent=None):
        self.name = name
        self.children = children
        self.parent = parent
        self.static_fullpath = static_fullpath
        self.path_type = None

    @property
    def exists(self):
        return os.path.exists(self.static_fullpath)

    @property
    def isdir(self):
        if self.exists:
            return os.path.isdir(self.static_fullpath)

    @property
    def isfile(self):
        if self.exists:
            return not self.isdir
    # @property
    def get_path_type(self):
        if self.isdir:
            self.path_type = 'folder'
            return self.path_type
        elif self.isfile:
            self.path_type = 'file'
            return self.path_type

    @property
    def ancestors(self):
        return [a for a in self.iter_up()]

    @property
    def ancestors_fullpath(self):
        """Similar to fullpath, but it's build from path ancestors"""
        return os.path.join(*[x.name for x in reversed(self.ancestors)])

    @property
    def root(self):
        return self.ancestors[-1]


    def iter_up(self):
        ''' Iterates upwards: yields self first, and ends with root
        Does not iterate over cousings or ancestors not in a direct inheritance
        line towards root
         '''
        yield self
        if self.parent is None:
            pass
        else:
            for parent in self.parent.iter_up():
                yield parent

    def iter_down(self):
        ''' Iterates downwards
        yields self first, then iterates over
        its children's children recursevely
        ending with last lowest child
        '''
        yield self
        for child in self.children:
            # yield child
            for c in child.iter_down():
                yield c


    def get_json_dict(self, detailed=False):
        d = OrderedDict()
        d['name'] = self.name
        d['type'] = self.path_type
        d['children'] = [x.get_json_dict() for x in self.children]
        if detailed:
            d['parent'] = getattr(self.parent, 'name', None)
        return d

    def get_json_string(self):
        return json.dumps(self.get_json_dict(), encoding='utf-8',
                          ensure_ascii=False, sort_keys=False, indent=2,
                          separators=(',', ': '))


    def __repr__(self):
        return '<PATH:{0}|PARENT:{1}|CHILDS:{2}>'.format(
                                            self.name,
                                            getattr(self.parent,'name', None),
                                            len(self))


    def __len__(self):
        '''Returns number of children, files or folders'''
        if self.children:
            return len(self.children)
        else:
            return 0


def tree_from_folder(source_folder):
    ''' creates patho tree of patho objects from a local folder name'''
    patho = PathO(os.path.basename(source_folder), static_fullpath=source_folder)
    patho.get_path_type()
    try:
        patho.children = [tree_from_folder(os.path.join(source_folder,x)) for x in os.listdir(source_folder)]
    except OSError as errmsg:
        pass # if is file, listdir will fail
    else:
        for child in patho.children:
            child.parent = patho
    return patho


def tree_from_json_dict(json_dict):
    ''' creates a PathO tree from a json_dict 
    (matching the jsson created by a PathO tree)'''
    patho = PathO(json_dict['name'])
    patho.path_type = json_dict['type']
    # import pdb; pdb.set_trace()
    try:
        patho.children = [tree_from_json_dict(x) for x in json_dict['children']]
    except KeyError:
        pass
    else:
        for child in patho.children:
            child.parent = patho
    return patho

core.py Functions to help with basic tasks, and manage exceptions

import json
import sys
import os
import shutil
import copy

from path_tree import tree_from_folder, tree_from_json_dict

# TO DO:
# Add Tests    

def load_file(source_file):
    ''' loads a json, returns a python list/dictionary object'''
    try:
        with open(source_file, 'r') as f:
            try:
                return json.load(f)
            except ValueError as e:
                print("Could Not Parse Json: {}".format(e))
    except IOError as errmsg:
        print errmsg
# tree = load_file('test.json')


def dump_json(filename, json_dict):
    ''' creates a .json file from a python object '''
    filename = '{0}.json'.format(filename)
    try:
        with open(filename, 'wx') as outfile:
            json.dump(json_dict, outfile, indent=2)
    except IOError as errmsg:
        print errmsg
    else:
        return True


# dump_json('test.json', tree.get_json_dict())


def mkdirs_from_json_dict(new_foldername, json_dict):
    if os.path.exists(new_foldername):
        print 'Cannot Copy. Folder already exists: ', new_foldername
        return

    new_tree = tree_from_json_dict(json_dict)
    new_tree.root.name = new_foldername     # set rootname of new folder

    failed = False
    for patho in new_tree.iter_down():
        if patho.path_type == 'folder':
            try:
                os.makedirs(patho.ancestors_fullpath)
            except OSError as errmsg:
                import pdb; pdb.set_trace()
                print errmsg
                failed = True
                break
        else:
            try:
                with open(patho.ancestors_fullpath, 'w') as f:
                    pass
            except OSError as errmsg:
                print errmsg
                failed = True
                break

    if not failed and os.path.exists(new_tree.root.ancestors_fullpath):
        return True
    else:
        print 'Make Dirs Operation Failed Deleting tree: ', new_foldername
        try:
            shutil.rmtree(new_tree.root.fullpath)
        except:
            print 'Attempted but failed to delete folder: ', new_foldername


def mkjson_from_folder(source_folder):
    """Returns Json_dict from folder, None if Folder not found or error"""
    if os.path.isdir(source_folder):
        tree = tree_from_folder(source_folder)
        return tree.get_json_dict()
    else:
        print 'Failed to make json. Folder not found: [{}]'.format(source_folder)

foldify.py Run this to see tools in action: a command line interface for the user.

import sys
from collections import OrderedDict

from core import mkjson_from_folder, mkdirs_from_json_dict
from core import load_file, dump_json


def menu_copy_folder_tree():
    """Copy Folder Tree."""
    source_folder = raw_input('Name of Source Folder: \n>>>')
    dest_folder = raw_input('Name of Destination Folder (Blank for X-copy): \n>>>')
    if dest_folder == '':
        dest_folder = '{}_copy'.format(source_folder)
    json_dict = mkjson_from_folder(source_folder)
    if json_dict and mkdirs_from_json_dict(dest_folder, json_dict):
        print 'Folder Structure of [{0}] successfully copied to [{1}]'.format(
                                            source_folder, dest_folder)


def menu_json_from_folder():
    """Make Json from Folder."""
    source_folder = raw_input('Name of Source Folder: \n>>>')
    dest_file = raw_input('Name of JSON (.json will be added; blank for same name): \n>>>')
    if dest_file == '':
        dest_file = source_folder

    json_dict = mkjson_from_folder(source_folder)
    if json_dict and dump_json(dest_file, json_dict):
        print 'New Json template created for folder [{}]'.format(source_folder)


def menu_folder_from_json():
    """Make Folder from Json."""
    source_file = raw_input('Name of Source JSON: \n>>>')
    dest_folder = raw_input('Name of Destination Folder (Leave Blank to try use json root): \n>>>')
    json_dict = load_file(source_file)
    if dest_folder == '':
        dest_folder = json_dict['name']
    if json_dict and mkdirs_from_json_dict(dest_folder, json_dict):
        print 'New folder [{}] created from json [{}]'.format(dest_folder,
                                                             source_file)



def menu_exit():
    """Exit the program."""
    sys.exit()


menu = (
    ('1', menu_copy_folder_tree),
    ('2', menu_json_from_folder),
    ('3', menu_folder_from_json),
    ('4', menu_exit)
        )
menu = OrderedDict(menu)

while True:
    print '='*30
    print 'Foldify'
    print '='*30
    for n, func in menu.items():
        print '{0} - {1}'.format(n, func.__doc__)
    selection = raw_input('Select an option:')
    try:
        menu[selection]()
    except KeyError:
        print 'Invalid Option'

Answer 1

You need to re-think your design. Make two halves of this, one is the API and should be programmed like an API, the other is the client code that uses this API. Do not ever mix the two. You have prints in the API part of the code that should really be in the client code part.

First off, you should formalize how you make your tree. I'd recommend that you convert the file system to the same way you would read the JSON. This is actually really simple. You get the name, and type of the node that you are at. You then get the nodes children if it has any.

And so I'd do something like:

# Globals
class FT: # FileType
    FOLDER = 'folder'
    FILE = 'file'
    OTHER = 'N/A'

class DN: # DataName
    NAME = 'name'
    TYPE = 'type'
    CHILDREN = 'children'


def _get_type(path):
    exists = os.path.exists
    isdir = os.path.isdir
    if not exists(path):
        return FT.OTHER
    if isdir(path):
        return FT.FOLDER
    return FT.FILE

def _read_path(path):
    obj = {
        DN.NAME: os.path.basename(path),
        DN.TYPE: _get_type(path)
    }
    if obj[DN.TYPE] == FT.FOLDER:
        join = os.path.join
        obj[DN.CHILDREN] = [
            _read_path(join(path, file_name))
            for file_name in os.listdir(path)
        ]
    return obj

This will give the same output as if we open the JSON file. Which can be simplified to:

def _read_json(path):
    with open(path, 'r') as f:
        return json.load(f)

After formalizing the input to the tree, you can now make the tree. The tree that you are making is most likely a B-Tree and so I'd call it that. I'm keeping things simple, as your code is overcomplicated to the point that I'd recommend re-writing it.

Anyway, the Node can create all it's children as it'll simplify the creation to just passing the object we get from the above functions. And then we want to be able to get a dict from it. And so the nodes can be:

class Node(object):
    def __init__(self, data):
        self.name = data.get(DN.NAME, None)
        self.parent = None
        self.type = data.get(DN.TYPE, FT.OTHER)
        self.children = [
            Node(child)
            for child in data.get(DN.CHILDREN, [])
        ]
        for child in self.children:
            child.parent = self

    def dict(self):
        return OrderedDict({
            DN.NAME: self.name,
            DN.TYPE: self.type,
            DN.CHILDREN: [c.dict() for c in self.children]
        })

After this I'd make a helper class to interact with the nodes. This class should also allow a way to save the nodes either as JSON or in the file system. Both can be simple.

class BTree(object):
    def __init__(self, path, json=False):
        fn = _read_json if json else _read_path
        data = fn(path)
        self.root = Node(data)

    def write_json(self, path):
        with open(path, 'w') as f:
            json.dump(self.root.dict(), f)

    def write_files(self, path):
        join = os.path.join
        path, name = os.path.split(path)
        self.root.name = name
        def build(path, node):
            path = join(path, node.name)
            if node.type == FT.FOLDER:
                os.makedirs(path)
            elif node.type == FT.FILE:
                with open(path, 'a') as f:
                    pass

            for child in node.children:
                build(path, child)
        build(path, self.root)

This now makes usage much easier than before. BTree(src).write_json(dest). You can then add more to the BTree to allow changes to the data. And if you want to change the data, only do it through BTree and Node. Do not change it when you load or save the data, these should be pure 1:1 changes of the Tree to the desired format. (And so I'd remove the two lines that do this in write_files, but I kept them to be consistent with your current code.)

Finally you can then simplify their usage in you client code. menu_copy_folder_tree, without error handling, can be:

def menu_copy_folder_tree():
    """Copy Folder Tree."""
    src = raw_input('Name of Source Folder: \n>>>')
    dest = raw_input('Name of Destination Folder (Blank for X-copy): \n>>>')
    if dest == '':
        dest = '{}_copy'.format(src)
    BTree(src).write_files(dest)
    print 'Folder Structure of [{0}] successfully copied to [{1}]'.format(src, dest)

You really need to aim for a less is more approach and separate program from API.