LXC Bootstrap - A wrapper script around lxc utility scripts to create and set up an LXC container based on settings

Question

So some background here.

I recently discovered the ease of LXC containers on Ubuntu. I've been building some 'test' containers locally for testing things I develop on Ubuntu, but the evil thing is that creating these containers then getting all the software I want in the container and everything set up the way I want it to be set up is not the most trivial of processes. First you have to create the container, then start the container, then run a long lxc-attach -n CONTAINERNAME -- [command] string depending on the command you want to execute (unless you want to drop into a shell, in which case you omit the -- [command] part), then you have to set up the things by hand with commands in the container.

Well, I found this quite annoying, as I need a 'base container' with Ubuntu Server in a userspace-usable format, which there is no 'base template' for. So, I wrote this, what I call lxc_bootstrap.py, which creates the container, starts the container, installs the packages I want on the image, sets up users per my needs, and then is done, and the container is running and ready for use.

And it's got a set of arguments that can be parsed to override standard bootstrap settings, such as different OSes, different release versions, different architectures, different packages or users to "bootstrap" the system with, etc. It can also run the bootstrapping process on existing containers, without creating a new one, with a specific flag.

Any suggestions for improvement are welcome, though I'm definitely using a lot of code that can probably be condensed or otherwise optimized, so such suggestions are most welcome above most other code suggestions. The code is here below.

Oh, and since I'm so picky about PEP8 style guidelines, I've got Flake8 running in CI from the repository, so it catches most evils. Note that per PEP8, a team/individual can choose to use a longer line length than 80 but no more than 120; I'm using a max line length of 120 in this case, so please don't call me out on that.

---

lxc_bootstrap.py:

#!/usr/bin/python

# lxc_bootstrap
# Copyright (C) 2017  Thomas Ward <teward@ubuntu.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# LXC Bootstrapper, around the lxc-create 'Download' template for userspace
# containers; creates then modifies the container based on specifications.
#
# Designed for Ubuntu / Debian systems.

# import os
import sys
import argparse
import crypt
import subprocess as sp
import random
import platform
from string import ascii_letters, digits

SHADOW_SALT_CHARSET = ascii_letters + digits

ARCHITECTURE_MAP = {
    'amd64': 'amd64',  # Explicit definition
    'i386': 'i386',    # Explicit definition
    'armhf': 'armhf',  # Explicit definition
    'arm64': 'arm64',  # Explicit definition
    'x86_64': 'amd64',
    'x86': 'i386',
    'armv7l': 'armhf',
    'armv8l': 'arm64'
}


# noinspection PyClassHasNoInit
class Bootstrap:
    # noinspection PyClassHasNoInit
    class Container:
        bootstrap_existing = False
        name = None
        architecture = None
        distribution = None
        release = None

    # noinspection PyClassHasNoInit
    class Customization:
        # noinspection PyClassHasNoInit
        class Packages:
            to_add = ['openssh-server', 'software-properties-common', 'haveged', 'python', 'python-dev',
                      'python3', 'python3-dev', 'perl-modules', 'ubuntu-server', 'iptables', 'libnetfilter-conntrack3']
            to_remove = ['lxd', 'lxd-client', 'lxd-tools', 'lxc']
            autoremove_after = False

        # noinspection PyClassHasNoInit
        class Users:
            userdata = [('teward', 'REDACTED', None, True)]


def _parse_arguments():
    argparser = argparse.ArgumentParser(description="LXC Container Bootstrapper Assistant", add_help=True)
    argparser.add_argument('-e', '--existing', '--bootstrap-existing', dest="container_bootstrap_existing",
                           default=False, required=False, action='store_true',
                           help="Don't create a container, run bootstrapping on "
                                "an already-existing container.")
    argparser.add_argument('-n', '--name', type=str, dest="container_name", required=True,
                           help="The name to assign to the LXC container.")
    argparser.add_argument('-a', '--arch', type=str, dest="container_arch", default=None, required=False,
                           help="The architecture for the container")
    argparser.add_argument('-d', '--dist', '--distro', type=str, dest="container_dist", default=None, required=False,
                           help="The distribution for the container")
    argparser.add_argument('-r', '--release', '--codename', type=str, dest="container_release", default=None,
                           required=False, help="The specific release of the container")
    argparser.add_argument('--add-packages', type=str, dest="packages_add", required=False,
                           default='openssh-server,software-properties-common,haveged,python,python-dev,'
                                   'python3,python3-dev,perl-modules,ubuntu-server,iptables',
                           help="Comma-separated list of packages to add to the container.")
    argparser.add_argument('--exclude-packages', type=str, dest="packages_exclude", required=False,
                           default='lxd,lxd-client,lxd-tools,lxc',
                           help="Comma-separated list of packages to exclude from the container.")
    argparser.add_argument('--users', '--userdata', '--userfile', type=str, dest="user_datafile", required=False,
                           default=None,
                           help="Path to a file containing user data, one user per line in USERNAME:PASSWORD:SALT:ADMIN"
                                " format, where SALT is an optional 8-character alpha numeric string, and ADMIN is "
                                "'True' or 'False'")

    return argparser.parse_args()


# noinspection PyShadowingNames
def _get_bootstrap(args):
    if type(args) is not argparse.Namespace:
        raise TypeError("Invalid arguments provided.")
    tmpbootstrap = Bootstrap()

    tmpbootstrap.Container.name = str(args.container_name).strip('\'')

    tmpbootstrap.Container.bootstrap_existing = args.container_bootstrap_existing

    if not tmpbootstrap.Container.bootstrap_existing:
        if 'Windows' in platform.platform():
            raise OSError("LXC doesn't work on Windows, so we can't use this script. Sorry!")
        elif 'Linux' not in platform.platform():
            raise OSError("This script only works for Linux OSes, sorry!")

        if args.container_arch:
            tmpbootstrap.Container.architecture = args.container_arch
        else:
            tmpbootstrap.Container.architecture = ARCHITECTURE_MAP[platform.machine()]

        if args.container_dist:
            tmpbootstrap.Container.distribution = args.container_dist
        else:
            lsb_dist = sp.Popen('/usr/bin/lsb_release -s -i'.split(), stdout=sp.PIPE, stderr=sp.PIPE).communicate()
            if lsb_dist[1]:
                raise SystemError("Error getting release distributor ID.")
            else:
                tmpbootstrap.Container.distribution = str(lsb_dist[0]).lower().strip('\r\n')

        if args.container_release:
            tmpbootstrap.Container.release = args.container_release
        else:
            try:
                lsb_codename = sp.Popen('/usr/bin/lsb_release -s -c'.split(),
                                        stdout=sp.PIPE, stderr=sp.PIPE).communicate()
                if lsb_codename[1]:
                    raise SystemError("Error getting release codename from lsb_release")
                tmpbootstrap.Container.release = str(lsb_codename[0]).lower().strip('\r\n')
            except Exception as e:
                raise e

    if args.packages_add:
        addpackages = args.packages_add.split(',')
        for pkg in addpackages:
            if pkg.lower() not in tmpbootstrap.Customization.Packages.to_add:
                tmpbootstrap.Customization.Packages.to_add.append(pkg.lower())

    if args.packages_exclude:
        delpackages = args.packages_exclude.split(',')
        for pkg in delpackages:
            if pkg.lower() in tmpbootstrap.Customization.Packages.to_add:
                tmpbootstrap.Customization.Packages.to_add.remove(pkg.lower())
            else:
                if pkg.lower() not in tmpbootstrap.Customization.Packages.to_remove:
                    tmpbootstrap.Customization.Packages.to_remove.append(pkg.lower())

    if args.user_datafile:
        datafile = open(args.user_datafile, 'r')
        for data in datafile:
            data = data.split(':')
            if len(data) != 4:
                raise IOError("Invalid data provided from datafile.")
            else:
                datatuple = (data[0], data[1], data[2], bool(data[3]))
                # noinspection PyTypeChecker
                tmpbootstrap.Customization.Users.userdata.append(datatuple)

    return tmpbootstrap


def _create_container(data):
    lxc_create_cmd = str("/usr/bin/lxc-create -t download -n %s -- -d %s -r %s -a %s" % (
        data.name, data.distribution, data.release, data.architecture)).split()

    try:
        # noinspection PyUnusedLocal
        ecode = sp.check_call(lxc_create_cmd, stdout=sys.stdout, stderr=sys.stderr)
    except sp.CalledProcessError:
        print "Something went wrong when creating the container."
        exit()


def _start_container(container_name):
    execstr = str('lxc-start -n %s' % container_name).split()
    try:
        # noinspection PyUnusedLocal
        ecode = sp.check_call(execstr, stdout=sys.stdout, stderr=sys.stderr)
    except sp.CalledProcessError:
        print "Could not start the container, cannot continue with bootstrap."
        exit()


def _shadow_pw_str(password, salt=None):
    if not salt or len(salt) != 8:
        # Create a random salt
        for _ in range(8):
            try:
                salt += random.SystemRandom().choice(SHADOW_SALT_CHARSET)
            except (TypeError, UnboundLocalError):
                salt = random.SystemRandom().choice(SHADOW_SALT_CHARSET)

    hashed = crypt.crypt(password, ('$6$%s$' % salt))
    return hashed


# noinspection PyUnreachableCode,PyUnusedLocal
def _container_bootstrap_users(container_name, data):
    # Default comes with 'ubuntu' user and group; let's nuke them
    del_default_usr = "/usr/bin/lxc-attach -n %s -- deluser --remove-all-files ubuntu" % container_name
    del_default_grp = "/usr/bin/lxc-attach -n %s -- delgroup --only-if-empty ubuntu" % container_name

    try:
        deldefusr = sp.check_call(del_default_usr.split(), stdout=sys.stdout, stderr=sys.stderr)
    except sp.CalledProcessError:
        print "Could not delete default user and/or group, please refer to error logs."

    try:
        deldefgrp = sp.check_call(del_default_grp.split(), stdout=sys.stdout, stderr=sys.stderr)
    except sp.CalledProcessError as e:
        if e.returncode == 5:
            print "Could not delete default group, it's not empty!"
        elif e.returncode == 3:
            pass  # return code of 3 means that the group doesn't exist, so we can move on.

    # Since we just nuked the '1000' group and user (default Ubuntu), let's start there now.
    uid = 1000
    gid = 1000

    for (username, password, salt, admin) in data.userdata:
        hashpw = _shadow_pw_str(password, salt)

        groupcreatecmd = "lxc-attach -n %s -- groupadd -g %s %s" % (container_name, gid, username)

        usrcreatecmd = ("lxc-attach -n %s -- useradd --create-home -u %s -g %s -p '%s' --shell=/bin/bash %s"
                        % (container_name, uid, gid, hashpw, username))

        try:
            groupcmd = sp.check_call(groupcreatecmd.split(), stdout=sys.stdout, stderr=sys.stderr)
            usercmd = sp.check_call(usrcreatecmd.split(), stdout=sys.stdout, stderr=sys.stderr)
            if admin:
                usermodcmd = "lxc-attach -n %s -- usermod -a -G sudo %s" % (container_name, username)
                usermod = sp.check_call(usermodcmd.split(), stdout=sys.stdout, stderr=sys.stderr)

        except sp.CalledProcessError:
            print "Something went wrong when bootstrapping user '%s'..." % username

        uid += 1
        gid += 1


# noinspection PyUnusedLocal
def _container_bootstrap_packages(container_name, data, autoremove=False):
    # Construct string
    basecommand = '/usr/bin/lxc-attach -n %s -- apt-get ' % container_name
    installstr = basecommand + "install -y %s" % " ".join(data.to_add)
    removestr = basecommand + "remove -y --purge %s" % " ".join(data.to_remove)
    autoremove = basecommand + "autoremove -y --purge"

    try:
        ecode = sp.call(installstr.split(), stdout=sys.stdout, stderr=sys.stderr)
    except sp.CalledProcessError:
        print "Something went wrong installing additional packages."
        exit()

    try:
        ecode = sp.call(removestr.split(), stdout=sys.stdout, stderr=sys.stderr)
    except sp.CalledProcessError:
        print "Something went wrong removing specified packages."
        exit()

    if autoremove:
        try:
            ecode = sp.call(autoremove.split(), stdout=sys.stdout, stderr=sys.stderr)
        except sp.CalledProcessError:
            print "Something went wrong cleaning up after removal with 'autoremove'."


def run():
    # Get args first, we'll pass this to the Bootstrapper.
    # noinspection PyUnreachableCode
    args = _parse_arguments()

    # Take the arguments, throw it into _get_bootstrap, and get a Bootstrap object.
    bootstrap = _get_bootstrap(args)

    if not bootstrap.Container.bootstrap_existing:
        # Create the container
        _create_container(bootstrap.Container)
    else:
        # Do nothing, continue on with bootstrap process.
        pass

    _start_container(bootstrap.Container.name)

    _container_bootstrap_packages(bootstrap.Container.name, bootstrap.Customization.Packages)

    _container_bootstrap_users(bootstrap.Container.name, bootstrap.Customization.Users)


if __name__ == "__main__":
    run()

Answer 1

I think you are using classes wrong. You use them only to have nice namespaces and do all the stuff you would normally do as methods outside of the class. And then you suppress the warnings about non-existing constructors...

In my opinion, you should have a User class that stores the info of a user and has a @property that returns the shadow password. Then, there is a Container class, that actually does the stuff. You don't need the Bootstrap, Customization, Packages classes at all.

Going from top to bottom, here are the changes I made to your code:

I defined a TransparentDict class that passes undefined keys right through, but returns the specified value for the defined keys.

The User class started life as a collections.namedtuple, but is a full class now, so it can have the shadow_password property.

Next, the most important part, the Container class. It has a __call__ method defined, which allows running arbitrary commands in the container and possibly giving it a fail string.

I also used str.format instead of the old % formatting throughout.

Almost all stand-alone functions of your code are now methods of this class. It can create and/or start a container, create the users and install packages.

The _parse_arguments does now some operation on the parsed args before returning them, so we have sensible things in there.

The run function is now slightly longer, because some stuff is in there, but in my opinion more readable now.

Bug:

Your code has the subtle bug that all users will be admins. This is because in Python all non-empty strings are truthy:

>>> bool("True")
True
>>> bool("False")
True

In my original code I had this turned around, by calling int and not bool, thinking you had a 0 or 1 in your userfile. However, you have a string "True" or "False" there. This will raise a ValueError for all users except the default user:

>>> int("True")
ValueError: invalid literal for int() with base 10: 'True'

To fix this, I now compare the admin parameter with both True and "True".

Final code:

#!/usr/bin/python

# lxc_bootstrap
# Copyright (C) 2017  Thomas Ward <teward@ubuntu.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# LXC Bootstrapper, around the lxc-create 'Download' template for userspace
# containers; creates then modifies the container based on specifications.
#
# Designed for Ubuntu / Debian systems.

# import os
import sys
import argparse
import crypt
import subprocess as sp
import random
import platform
from string import ascii_letters, digits

SHADOW_SALT_CHARSET = ascii_letters + digits

RANDOM_ENGINE = random.SystemRandom()


class TransparentDict(dict):

    def __missing__(self, key):
        return key


ARCHITECTURE_MAP = TransparentDict({
    'x86_64': 'amd64',
    'x86': 'i386',
    'armv7l': 'armhf',
    'armv8l': 'arm64'
})


class User:

    def __init__(self, name, password, salt, admin):
        self.name = name
        self.password = password
        self.salt = salt
        if not self.salt or len(self.salt) != 8:
            # Create a random salt
            self.salt = "".join(RANDOM_ENGINE.choice(SHADOW_SALT_CHARSET) for _ in range(8))
        self.admin = admin in (True, "True")

    @property
    def shadow_password(self):
        return crypt.crypt(self.password, ('$6${}$'.format(self.salt)))


class Container:
    create_cmd = "/usr/bin/lxc-create -t download -n {0.name} -- -d {0.distribution} -r {0.release} -a {0.architecture}"
    start_cmd = "lxc-start -n {0.name}"
    attach_cmd = "/usr/bin/lxc-attach -n {0.name} -- "

    def __init__(self, name, architecture, distribution, release):
        self.name = name
        self.architecture = architecture
        self.distribution = distribution
        self.release = release
        self.attach = self.attach_cmd.format(self).split()

    def __call__(self, cmd, error_msg="", attach=True):
        cmd = cmd.split()
        if attach:
            cmd = self.attach + cmd
        try:
            sp.check_call(cmd, stdout=sys.stdout, stderr=sys.stderr)
        except sp.CalledProcessError as e:
            if error_msg:
                print error_msg
                print e
                sys.exit()
            raise e

    def create(self):
        cmd = self.create_cmd.format(self)
        self(cmd, "Something went wrong when creating the container.", attach=False)

    def start(self):
        cmd = self.start_cmd.format(self)
        self(cmd, "Could not start the container, cannot continue with bootstrap.", attach=False)

    def bootstrap_users(self, users):
        # Default comes with 'ubuntu' user and group; let's nuke them
        self("deluser --remove-all-files {}".format("ubuntu"),
             "Could not delete default user and/or group, please refer to error logs.")
        try:
            self("delgroup --only-if-empty {}".format("ubuntu"))
        except sp.CalledProcessError as e:
            if e.returncode == 5:
                print "Could not delete default group, it's not empty!"
            elif e.returncode == 3:
                # return code of 3 means that the group doesn't exist, so we can
                # move on.
                pass

        # Since we just nuked the '1000' group and user (default Ubuntu), let's
        # start there now.
        uid, gid = 1000, 1000

        for user in users:
            try:
                self("useradd --create-home -u {uid} -g {gid} -p {user.shadow_password} --shell=/bin/bash {user.name}".format(
                    user=user, gid=gid, uid=uid))
                self(
                    "groupadd -g {gid} {user.name}".format(user=user, gid=gid))
                if user.admin:
                    self("usermod -a -G sudo {user.name}".format(user=user))
            except sp.CalledProcessError:
                print "Something went wrong when bootstrapping user '{0.name}'...".format(user)
            uid += 1
            gid += 1

    def bootstrap_packages(self, to_add, to_exclude, autoremove=False):
        self("apt-get install -y {}".format(" ".join(to_add)), "Something went wrong installing additional packages.")
        self("apt-get remove -y --purge {}".format(" ".join(to_remove)), "Something went wrong removing specified packages.")
        if autoremove:
            self("apt-get autoremove -y --purge", "Something went wrong cleaning up after removal with 'autoremove'.")


def _parse_arguments():
    current_platform = platform.platform()
    if 'Windows' in current_platform:
        raise OSError(
            "LXC doesn't work on Windows, so we can't use this script. Sorry!")
    elif 'Linux' not in current_platform:
        raise OSError("This script only works for Linux OSes, sorry!")

    argparser = argparse.ArgumentParser(
        description="LXC Container Bootstrapper Assistant", add_help=True)
    argparser.add_argument('-e', '--existing', '--bootstrap-existing', dest="container_bootstrap_existing",
                           default=False, required=False, action='store_true',
                           help="Don't create a container, run bootstrapping on "
                           "an already-existing container.")
    argparser.add_argument('-n', '--name', type=str, dest="container_name", required=True,
                           help="The name to assign to the LXC container.")
    argparser.add_argument('-a', '--arch', type=str, dest="container_arch", default=None, required=False,
                           help="The architecture for the container")
    argparser.add_argument('-d', '--dist', '--distro', type=str, dest="container_dist", default=None, required=False,
                           help="The distribution for the container")
    argparser.add_argument('-r', '--release', '--codename', type=str, dest="container_release", default=None,
                           required=False, help="The specific release of the container")
    argparser.add_argument('--add-packages', type=str, dest="packages_add", required=False,
                           default='openssh-server,software-properties-common,haveged,python,python-dev,'
                           'python3,python3-dev,perl-modules,ubuntu-server,iptables',
                           help="Comma-separated list of packages to add to the container.")
    argparser.add_argument('--exclude-packages', type=str, dest="packages_exclude", required=False,
                           default='lxd,lxd-client,lxd-tools,lxc',
                           help="Comma-separated list of packages to exclude from the container.")
    argparser.add_argument('--users', '--userdata', '--userfile', type=str, dest="user_datafile", required=False,
                           default=None,
                           help="Path to a file containing user data, one user per line in USERNAME:PASSWORD:SALT:ADMIN"
                           " format, where SALT is an optional 8-character alpha numeric string, and ADMIN is "
                           "'True' or 'False'")

    args = argparser.parse_args()
    args.container_name = args.container_name.strip('\'')

    if not args.container_bootstrap_existing:
        if not args.container_arch:
            args.container_arch = ARCHITECTURE_MAP[
                platform.machine()]

        if not args.container_dist:
            lsb_dist = sp.Popen('/usr/bin/lsb_release -s -i'.split(),
                                stdout=sp.PIPE, stderr=sp.PIPE).communicate()
            if lsb_dist[1]:
                raise SystemError("Error getting release distributor ID.")
            else:
                args.container_dist = lsb_dist[0].lower().strip('\r\n')

        if not args.container_release:
            lsb_codename = sp.Popen('/usr/bin/lsb_release -s -c'.split(),
                                    stdout=sp.PIPE, stderr=sp.PIPE).communicate()
            if lsb_codename[1]:
                raise SystemError(
                    "Error getting release codename from lsb_release")
            args.container_release = lsb_codename[0].lower().strip('\r\n')

    args.to_add = set()
    args.to_remove = set()
    args.autoremove_after = False
    if args.packages_add:
        args.to_add |= set(map(str.lower, args.packages_add.split(',')))

    if args.packages_exclude:
        delpackages = set(map(str.lower, args.packages_exclude.split(',')))
        args.to_add -= delpackages
        args.to_remove |= delpackages
    return args


def get_users(user_datafile):
    users = []
    if user_datafile:
        with open(user_datafile) as datafile:
            users = [User(*row.split(':')) for row in datafile]
    return users


def run():
    args = _parse_arguments()
    container = Container(args.container_name,
                          args.container_arch,
                          args.container_dist,
                          args.container_release)

    users = [User('teward', 'REDACTED', None, True)]2
    users += get_users(args.user_datafile)

    if not args.container_bootstrap_existing:
        container.create()

    container.start()
    container.bootstrap_users(users)
    container.bootstrap_packages(args.to_add, args.to_exclude)


if __name__ == "__main__":
    run()