Long running Python “server status” collector

Question

I wrote a small program that collects the server status, caches it for a certain amount of seconds, and then sends that to a mothership server. It's broken down into a few pieces, starting with main.py, config.json, and a few small 'libs' to keep things visually separate:

main.py

#!/usr/bin/python
import os
import sys
import json
import time
import sched
import socket
import psutil
from lib import cpu, memory, disks, network, system, transport

__cache = []
__cache_timer = 0
__cache_keeper = 0

def main(scheduler, config, sock, hostname, callers):
    global __cache
    global __cache_timer
    global __cache_keeper

    payload = {
        "_id": {
            "time": time.time(),
            "id": config['identification']['id'],
            "hostname": hostname,
            "type": config['identification']['type']
        },
        "cpu": callers['cpu'].snapshot(),
        "memory": callers['memory'].snapshot(),
        "disks": callers['disks'].snapshot(),
        "network": callers['network'].snapshot(),
        "system": callers['system'].snapshot()
    }
    __cache.append(payload)

    if __cache_keeper < __cache_timer:
        __cache_keeper += config['interval']
    else:
        transport.Transport({"payload": json.dumps(__cache)}, config, sock)
        __cache_keeper = 0
        __cache = []

    # Schedule a new run at the specified interval
    scheduler.enter(config['interval'], 1, main, (scheduler, config, sock, hostname, callers))
    scheduler.run()

if __name__ == '__main__':
    try:
        config = (json.loads(open(os.path.dirname(os.path.abspath(__file__)) + "/config.json").read()))['config']
        config['identification']['type'] = config['identification'].get('type', 'false')

        config['disable_cache'] = False
        if config['cache'].get('enabled') is True:
            __cache_timer = config['cache'].get('time_seconds_to_cache_between_sends', 60)
            config['interval'] = config['cache'].get('interval_seconds_between_captures', 5)

            # If the interval is higher, just exit
            if config['interval'] > __cache_timer:
                print >> sys.stderr, "Report interval is higher than cache timer."
                sys.exit(1)

        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        scheduler = sched.scheduler(time.time, time.sleep)
        hostname = config['identification'].get('hostname', socket.gethostname())
        callers = {
            "cpu": cpu.CPU(psutil),
            "memory": memory.Memory(psutil),
            "disks": disks.Disks(psutil),
            "network": network.Network(psutil),
            "system": system.System(psutil)
        }
        main(scheduler, config, sock, hostname, callers)
    except KeyboardInterrupt:
        print >> sys.stderr, '\nExiting by user request.\n'
        sys.exit(0)
    except Exception as e:
        location = '\n' + type(e).__name__
        print >> sys.stderr, location, '=>', str(e)
        sys.exit(1)

config.json

{
    "config": {
        "mothership": {
            "host": "127.0.0.1",
            "port": 1336
        },
        "cache": {
            "enabled": true,
            "time_seconds_to_cache_between_sends": 5,
            "interval_seconds_between_captures": 1
        },
        "identification": {
            "id": "some-id-here"
        }
    }
}

lib/cpu.py

from subprocess import check_output

class CPU:
    psutil = None
    cpu_count = {}
    cpu_passthrough = 0

    def __init__(self, psutil):
        self.psutil = psutil

    def snapshot(self):
        """
        Generate a snapshot of the current CPU state
        """
        cpu_time = self.psutil.cpu_times()

        # Only update the CPU counts every 100th pass through
        if self.cpu_count == {} or self.cpu_passthrough % 100 == 0:
            self.cpu_count = {
                "virtual": self.psutil.cpu_count(),
                "physical": self.psutil.cpu_count(logical=False)
            }

        return {
            "cpu_percent": self.psutil.cpu_percent(interval=1, percpu=True),
            "cpu_times": {
                "user": cpu_time[0],
                "system": cpu_time[1],
                "idle": cpu_time[2]
            },
            "cpu_count": self.cpu_count,
            "load_average": check_output(["cat", "/proc/loadavg"])
        }

lib/disks.py

class Disks:
    psutil = None
    disk_usage = None
    disk_partitions = None
    disk_passthrough = 0

    def __init__(self, psutil):
        self.psutil = psutil

    def snapshot(self):
        """
        Generate a snapshot of the current disk state
        """
        # Only grab the disk partitions every 25th pass
        if self.disk_partitions is None or self.disk_passthrough % 25 == 0:
            self.disk_partitions = self.psutil.disk_partitions(all=True)

        # Only grab the disk usage every 5th pass
        if self.disk_usage is None or self.disk_passthrough % 5 == 0:
            self.disk_usage = self.psutil.disk_usage('/')

        self.disk_passthrough += 1

        return {
            "disk_usage": self.disk_usage,
            "disk_partitions": self.disk_partitions,
            "disk_io_counters": self.psutil.disk_io_counters(perdisk=True)
        }

lib/memory.py

class Memory:
    psutil = None

    def __init__(self, psutil):
        self.psutil = psutil

    def snapshot(self):
        """
        Generate a snapshot of the current memory state
        """
        return {
            "virtual": self.psutil.virtual_memory(),
            "swap": self.psutil.swap_memory()
        }

lib/network.py

class Network:
    psutil = None

    def __init__(self, psutil):
        self.psutil = psutil

    def snapshot(self):
        """
        Generate a snapshot of the current network state
        """
        return {
            "net_io_counters": self.psutil.net_io_counters(pernic=True)
        }

lib/system.py

class System:
    psutil = None

    def __init__(self, psutil):
        self.psutil = psutil

    def snapshot(self):
        """
        Generate a snapshot of the current system state
        """
        return {
            "users": self.psutil.users(),
            "boot_time": self.psutil.boot_time()
        }

lib/transport.py

class Transport():
    def __init__(self, payload, config, sock):
        payload = str(payload)
        sock.setblocking(0)
        sock.sendto(payload, (config.get('mothership').get('host'), 
                              config.get('mothership').get('port')))

Have I done anything wrong? What could I improve?

Answer 1

Don't prefix names with two underscores; those are reserved for name mangling. Use a single underscore.

When opening files, use with:

with open(...) as my_file:
    ... # use my_file

This deals with cleanup.

Don't do this:

class MyClass:
    var = ...

The var here is actually a global variable. This is one of Python's less obvious gotcha's. Instead, assign it in the constructor.

I would be very wary of initializing data to wrong-but-valid-looking data. If possible, don't initialize early. If not, make it explicit - normally this means to use None.