Nagios status.dat to dict() - Could it be better?

Question

This is something myself and a former colleague were working on and I've finally finished it but wanting to see if we've done as well as we can or if it can be further optimised.

Also looking at doing a new feature for getting a list of all services rather than hosts and find the status of the service on each host.

Here's what I've got to so far:

import sys
import re
import os

def get_nagios_status(path):
    """
    Parse the Nagios status.dat file into a dict to give the ability to report on host/service information 

    `path` is the absolute path to the Nagios status.dat

    Return dict() if OK, -1 if not
    """

    result = {}
    record = {}
    mode     = ''
    parse_enabled = False

    try:
        f = open(path, 'r')
    except:
        return -1

    try:
        for line in f.readlines():
            line = line.rstrip()

            if re.match('^(\w+) \{$', line):
                if line.find('host') >= 0:
                    mode = 'host'
                elif line.find('service') >= 0 and not line.find('servicecomment') >= 0:
                    mode = 'service'
                elif line.find('info') >= 0:
                    mode = 'info'
                elif line.find('program') >= 0:
                    mode = 'program'
                else:
                    continue

                record = {}
                parse_enabled = True
                continue
            elif parse_enabled and re.match('^\t\}$', line):
                if mode == 'host':
                    if result.get(mode, None) is None:
                        result[mode] = {}
                    result[mode][record['host_name']] = record.copy()
                elif mode == 'service':
                    if result.get(mode, None) is None:
                        result[mode] = {}
                    if result[mode].get(record['host_name'], None) is None:
                        result[mode][record['host_name']] = {}
                    result[mode][record['host_name']][record['service_description']] = record.copy()
                else:
                    result[mode] = record.copy()
                parse_enabled = False
                continue
            elif re.match('^\t\w', line):
                pass
#           elif not parse_enabled or re.match('^(\s*)#', line):
#               continue
            else:
                continue

            data = line.strip().split('=', 1)
            record[data[0]] = data[1]

        f.close()

        return result
    except:
        return -1

# Get the status.dat file automatically...
for line in open('/etc/nagios/nagios.cfg').readlines():
    if line.startswith('status_file='):
        status_file = line.strip('\n').split("=")[1]
        break

status = get_nagios_status(status_file)

server = 'server1'
# Get information about a host
print status['host'][server]

# Find any issues on a hoot
for service in status['service'][server]:
    if int(status['service'][server][service]['current_state']) != 0:
        print(service)

==== EDIT: Below is an example input file (status.dat) ====

########################################
#          NAGIOS STATUS FILE
#
# THIS FILE IS AUTOMATICALLY GENERATED
# BY NAGIOS.  DO NOT MODIFY THIS FILE!
########################################

info {
    created=1399014558
    version=3.3.1
    last_update_check=1398994015
    update_available=1
    last_version=3.3.1
    new_version=4.0.6
    }

programstatus {
    modified_host_attributes=0
    modified_service_attributes=0
    nagios_pid=2953
    daemon_mode=1
    program_start=1398953444
    last_command_check=1399014554
    last_log_rotation=1398985200
    enable_notifications=1
    active_service_checks_enabled=1
    passive_service_checks_enabled=1
    active_host_checks_enabled=1
    passive_host_checks_enabled=1
    enable_event_handlers=1
    obsess_over_services=0
    obsess_over_hosts=0
    check_service_freshness=1
    check_host_freshness=0
    enable_flap_detection=1
    enable_failure_prediction=1
    process_performance_data=1
    global_host_event_handler=
    global_service_event_handler=
    next_comment_id=8239
    next_downtime_id=52
    next_event_id=1334236
    next_problem_id=590691
    next_notification_id=567984
    total_external_command_buffer_slots=4096
    used_external_command_buffer_slots=0
    high_external_command_buffer_slots=1
    active_scheduled_host_check_stats=107,394,1097
    active_ondemand_host_check_stats=15,95,286
    passive_host_check_stats=0,0,0
    active_scheduled_service_check_stats=465,2821,8607
    active_ondemand_service_check_stats=0,0,0
    passive_service_check_stats=0,0,0
    cached_host_check_stats=15,94,282
    cached_service_check_stats=0,0,0
    external_command_stats=0,0,0
    parallel_host_check_stats=107,395,1100
    serial_host_check_stats=0,0,0
    }

hoststatus {
    host_name=deep-thought
    modified_attributes=0
    check_command=check-host-alive
    check_period=24x7
    notification_period=24x7
    check_interval=1.000000
    retry_interval=1.000000
    event_handler=
    has_been_checked=1
    should_be_scheduled=1
    check_execution_time=4.015
    check_latency=4.569
    check_type=0
    current_state=0
    last_hard_state=0
    last_event_id=1180007
    current_event_id=1180013
    current_problem_id=0
    last_problem_id=522333
    plugin_output=PING OK - Packet loss = 0%, RTA = 0.75 ms
    long_plugin_output=
    performance_data=rta=0.755000ms;3000.000000;5000.000000;0.000000 pl=0%;80;100;0
    last_check=1399014471
    next_check=1399014546
    check_options=0
    current_attempt=1
    max_attempts=10
    state_type=1
    last_state_change=1391171572
    last_hard_state_change=1372696148
    last_time_up=1399014486
    last_time_down=1391171500
    last_time_unreachable=0
    last_notification=0
    next_notification=0
    no_more_notifications=0
    current_notification_number=0
    current_notification_id=0
    notifications_enabled=1
    problem_has_been_acknowledged=0
    acknowledgement_type=0
    active_checks_enabled=1
    passive_checks_enabled=1
    event_handler_enabled=1
    flap_detection_enabled=1
    failure_prediction_enabled=1
    process_performance_data=1
    obsess_over_host=1
    last_update=1399014558
    is_flapping=0
    percent_state_change=0.00
    scheduled_downtime_depth=0
    }

hoststatus {
    host_name=another-host
    modified_attributes=1
    check_command=check-host-alive
    check_period=24x7
    notification_period=24x7
    check_interval=1.000000
    retry_interval=1.000000
    event_handler=
    has_been_checked=1
    should_be_scheduled=1
    check_execution_time=30.005
    check_latency=2.607
    check_type=0
    current_state=1
    last_hard_state=1
    last_event_id=1329401
    current_event_id=1329409
    current_problem_id=588425
    last_problem_id=578077
    plugin_output=(Host Check Timed Out)
    long_plugin_output=
    performance_data=
    last_check=1399014431
    next_check=1399014529
    check_options=0
    current_attempt=1
    max_attempts=2
    state_type=1
    last_state_change=1398685630
    last_hard_state_change=1398685630
    last_time_up=1398685440
    last_time_down=1399014469
    last_time_unreachable=0
    last_notification=0
    next_notification=0
    no_more_notifications=0
    current_notification_number=0
    current_notification_id=51079
    notifications_enabled=0
    problem_has_been_acknowledged=0
    acknowledgement_type=0
    active_checks_enabled=1
    passive_checks_enabled=1
    event_handler_enabled=1
    flap_detection_enabled=1
    failure_prediction_enabled=1
    process_performance_data=1
    obsess_over_host=1
    last_update=1399014558
    is_flapping=0
    percent_state_change=0.00
    scheduled_downtime_depth=0
    }

servicestatus {
    host_name=deep-thought
    service_description=Application McAfee DAT Date
    modified_attributes=0
    check_command=check_windows_nrpe!check_av!3!5
    check_period=24x7
    notification_period=24x7
    check_interval=60.000000
    retry_interval=5.000000
    event_handler=
    has_been_checked=1
    should_be_scheduled=1
    check_execution_time=0.658
    check_latency=0.268
    check_type=0
    current_state=0
    last_hard_state=0
    last_event_id=1084911
    current_event_id=1084987
    current_problem_id=0
    last_problem_id=482213
    current_attempt=1
    max_attempts=3
    state_type=1
    last_state_change=1388710304
    last_hard_state_change=1372696148
    last_time_ok=1399014521
    last_time_warning=1388710009
    last_time_unknown=0
    last_time_critical=1372696455
    plugin_output=OK - DAT date (01/05/2014)
    long_plugin_output=
    performance_data=
    last_check=1399014521
    next_check=1399018121
    check_options=0
    current_notification_number=0
    current_notification_id=0
    last_notification=0
    next_notification=0
    no_more_notifications=0
    notifications_enabled=1
    active_checks_enabled=1
    passive_checks_enabled=1
    event_handler_enabled=1
    problem_has_been_acknowledged=0
    acknowledgement_type=0
    flap_detection_enabled=1
    failure_prediction_enabled=1
    process_performance_data=1
    obsess_over_service=1
    last_update=1399014558
    is_flapping=0
    percent_state_change=0.00
    scheduled_downtime_depth=0
    }

servicestatus {
    host_name=deep-thought
    service_description=Application Netbackup Version
    modified_attributes=0
    check_command=check_windows_nrpe!check_netbackup!6.5!6
    check_period=24x7
    notification_period=24x7
    check_interval=60.000000
    retry_interval=5.000000
    event_handler=
    has_been_checked=1
    should_be_scheduled=1
    check_execution_time=0.563
    check_latency=2.887
    check_type=0
    current_state=0
    last_hard_state=0
    last_event_id=557354
    current_event_id=557367
    current_problem_id=0
    last_problem_id=260970
    current_attempt=1
    max_attempts=3
    state_type=1
    last_state_change=1372696479
    last_hard_state_change=1372696148
    last_time_ok=1399011262
    last_time_warning=0
    last_time_unknown=0
    last_time_critical=1372696455
    plugin_output=Netbackup Not Installed!
    long_plugin_output=
    performance_data=
    last_check=1399011262
    next_check=1399014862
    check_options=0
    current_notification_number=0
    current_notification_id=0
    last_notification=0
    next_notification=0
    no_more_notifications=0
    notifications_enabled=1
    active_checks_enabled=1
    passive_checks_enabled=1
    event_handler_enabled=1
    problem_has_been_acknowledged=0
    acknowledgement_type=0
    flap_detection_enabled=1
    failure_prediction_enabled=1
    process_performance_data=1
    obsess_over_service=1
    last_update=1399014558
    is_flapping=0
    percent_state_change=0.00
    scheduled_downtime_depth=0
    }

servicestatus {
    host_name=another-host
    service_description=Application PowerPath Status
    modified_attributes=0
    check_command=check_windows_nrpe!check_powerpath!1
    check_period=24x7
    notification_period=24x7
    check_interval=1.000000
    retry_interval=1.000000
    event_handler=
    has_been_checked=1
    should_be_scheduled=1
    check_execution_time=0.525
    check_latency=2.873
    check_type=0
    current_state=0
    last_hard_state=0
    last_event_id=1179979
    current_event_id=1180016
    current_problem_id=0
    last_problem_id=522331
    current_attempt=1
    max_attempts=3
    state_type=1
    last_state_change=1391171576
    last_hard_state_change=1391171576
    last_time_ok=1399014497
    last_time_warning=0
    last_time_unknown=0
    last_time_critical=1391171512
    plugin_output=Power Path not installed on this machine
    long_plugin_output=
    performance_data=
    last_check=1399014497
    next_check=1399014557
    check_options=0
    current_notification_number=0
    current_notification_id=72470
    last_notification=0
    next_notification=0
    no_more_notifications=0
    notifications_enabled=1
    active_checks_enabled=1
    passive_checks_enabled=1
    event_handler_enabled=1
    problem_has_been_acknowledged=0
    acknowledgement_type=0
    flap_detection_enabled=1
    failure_prediction_enabled=1
    process_performance_data=1
    obsess_over_service=1
    last_update=1399014558
    is_flapping=0
    percent_state_change=0.00
    scheduled_downtime_depth=0
    }

servicestatus {
    host_name=another-host
    service_description=Application Windows Updates
    modified_attributes=0
    check_command=check_windows_nrpe!check_updates!/w:1!/c:2!/o:1!/x:2
    check_period=24x7
    notification_period=24x7
    check_interval=60.000000
    retry_interval=5.000000
    event_handler=
    has_been_checked=1
    should_be_scheduled=1
    check_execution_time=32.098
    check_latency=0.453
    check_type=0
    current_state=2
    last_hard_state=2
    last_event_id=1309596
    current_event_id=1309646
    current_problem_id=540922
    last_problem_id=260968
    current_attempt=3
    max_attempts=3
    state_type=1
    last_state_change=1397242439
    last_hard_state_change=1397242439
    last_time_ok=1392138849
    last_time_warning=1397238837
    last_time_unknown=1390021982
    last_time_critical=1399011650
    plugin_output=Number of critical updates not installed: 10 <br />Number of software updates not installed: 1 <br />  Critical updates name: Security Update for Internet Explorer 8 for Windows Server 2003 (KB2964358)+ Cumulative Security Update for Internet Explorer 8 for Windows Server 2003 (KB2936068)+ Windows Malicious Software Removal Tool - April 2014 (KB890830)+ Security Update for Windows Server 2003 (KB2922229)+ Security Update for Windows Server 2003 (KB2929961)+ Security Update for Windows Server 2003 (KB2930275)+ Security Update for Windows Server 2003 (KB2916036)+ Security Update for Microsoft .NET Framework 1.1 SP1 on Windows Server 2003 and Windows Server 2003 R2 x86 (KB2901115)+ Security Update for Microsoft .NET Framework 1.1 SP1 on Windows Server 2003 and Windows Server 2003 R2 x86 (KB2898860)+ Security Update for Windows Server 2003 (KB2909210)+  <br />Software Updates: Update for Windows Server 2003 (KB2927811)+
    long_plugin_output=Total No of Updates:11CritLvl: 2 WarnLvl:1OptCritLvl:OptWarnLvl:1\n
    performance_data=
    last_check=1399011650
    next_check=1399015250
    check_options=0
    current_notification_number=20
    current_notification_id=565252
    last_notification=1398972073
    next_notification=1399404073
    no_more_notifications=0
    notifications_enabled=1
    active_checks_enabled=1
    passive_checks_enabled=1
    event_handler_enabled=1
    problem_has_been_acknowledged=0
    acknowledgement_type=0
    flap_detection_enabled=1
    failure_prediction_enabled=1
    process_performance_data=1
    obsess_over_service=1
    last_update=1399014558
    is_flapping=0
    percent_state_change=0.00
    scheduled_downtime_depth=0
    }

contactstatus {
    contact_name=24x7
    modified_attributes=0
    modified_host_attributes=0
    modified_service_attributes=0
    host_notification_period=24x7
    service_notification_period=24x7
    last_host_notification=1397636581
    last_service_notification=1394546456
    host_notifications_enabled=1
    service_notifications_enabled=1
    }

servicecomment {
    host_name=deep-thought
    service_description=Application McAfee DAT Date
    entry_type=1
    comment_id=4238
    source=1
    persistent=1
    entry_time=1375437640
    expires=0
    expire_time=0
    author=nagiosadmin
    comment_data=This server is being replaced as it's failing.
    }

Answer 1

Unfortunately I was not able to get the code to run and produce any results even though I adjusted it to load the data file. It just returned an empty dict. Rather than try to figure out why, I took a look at the things I would change based on reading through the code and just made those changes, it seems to work after I did that, not really sure what the cause was but I did do the testing on Windows which may have had something to do with it.

Here are the things I focused on:

1. Use a context manager to do the file opening rather than the try except up top and then closing the file manually at the bottom.
2. Don't bother with f.readlines() just do for line in f:.
3. Stripped both sides of the line rather than the right side first and then the left side later.
4. Pulled out regular expressions and the parse flag, just use the mode as the flag, set it to None if the mode is not of interest.
5. Changed the if line.find(...) >= 0, just use if ... in line.... but it may be even easier to just lookup the "mode" in a list of "modes" you are tracking so I changed that.
6. I changed the mode naming to match the file. That was just to make things easier. If you really want them renamed perhaps the modes could be a dict rather than a list.
7. Replaced result.get(mode, None) is None: with if 'hoststatus' in mode: style check.
8. Before setting the values into the record, I checked to make sure there was something on the right side of the equal if len(data) > 1:. This may be something that the regular expressions avoid but it didn't run right for me.
9. At the bottom of your code you should put in the if __name__ == '__main__':.
10. There were a couple pep8 issues with line length. I just broke them down a bit.
11. Removed all imports since I didn't need them anymore.

As was already mentioned, I also am not crazy about -1 return, I would rather see it raise the exception rather than an odd exception later about integer subscript so I removed the try/except negative 1 return.

Here is the code I came up with after applying the changes mentioned above:

def get_nagios_status(path):
    """
    Parse the Nagios status.dat file into a dict to give the ability to report
    on host/service information

    `path` is the absolute path to the Nagios status.dat

    Return dict() if OK
    """
    result = {}
    mode = None
    modes = ['hoststatus', 'servicestatus', 'info', 'programstatus']

    with open(path, 'r') as f:
        for line in f:
            line = line.strip()

            # Mode starting
            if line.endswith('{'):
                mode = line.split()[0]

                if mode in modes:
                    record = {}
                    if mode not in result:
                        result[mode] = {}
                else:
                    mode = None

            # Mode ending
            elif line.endswith('}') and mode:
                if 'hoststatus' in mode:
                    result[mode][record['host_name']] = record.copy()

                elif 'servicestatus' in mode:
                    if record['host_name'] not in result[mode]:
                        result[mode][record['host_name']] = {}

                    result[mode][record['host_name']][
                            record['service_description']] = record.copy()
                else:
                    result[mode] = record.copy()

                mode = None

            # Collect the data if we are interested
            elif mode:
                data = line.split('=', 1)
                if len(data) > 1:
                    record[data[0]] = data[1]

    return result

Answer 2

Firstly, you should use the with context manager to handle your file:

with open(path) as f: # 'r' is the default
    for line in f.readlines():
        ...

This will handle any file-related errors and ensure the file is closed correctly.

Second, you could simplify your result dictionary handling with one of a few options:

1. if result.get(mode) is None: (None is the default for d);
2. Initialise as result = {'host': {}, 'service': {}, 'info': {}, 'program': {}}; or
3. Initialise as result =collections.defaultdict(dict).

Third, your main function is quite long. I would suggest you break a few bits out into smaller functions, e.g.

record.update(parse_data_line(data))

where

def parse_data_line(data):
    """Parse a single line of the file."""
    data = data.strip().split("", 1)
    return {data[0]: data[1]}

Another obvious candidate:

elif parse_enabled and re.match('^\t\}$', line):
    process_record(record, mode, result)

You can also use re more cleverly, and be more explicit about which parts of the file you want:

MODES = {"hoststatus": "host", "info": "info", ...} # define modes you want
...
matches = re.findall(r'^(\w+) \{$', line)
if matches:
    mode = MODES.get(matches[0])
    if mode is None:
        continue

(Note: option 2 above can now be result = {k: {} for k in MODES} to reduce duplication)

Now your main function looks like:

def get_nagios_status(path):
    MODES = {"hoststatus": "host", "info": "info", ...}
    result = {k: {} for k in MODES}
    parse_enabled = False
    with open(path) as f:
        for line in f.readlines():
            line = line.rstrip()
            matches = re.findall(r'^(\w+) \{$', line)
            if matches:
                mode = MODES.get(matches[0])
                if mode is None:
                    continue
                parse_enabled = True
                record = {}
            elif parse_enabled and re.match('^\t\}$', line):
                process_record(record, mode, result)
                parse_enabled = False
            elif parse_enabled and re.match('^\s*.*=.*$', line):
                record.update(parse_data_line(line))
   return result or -1

This is much more readable, partly as that function fits into one screen.

Answer 3

• Catching all exceptions and returning -1 is a bad idea. The exceptions contain information what exactly went wrong. Maybe the file was not accessible, or maybe you have a simple bug in your code, you can't tell.
• What's worse is that you don't even check for the -1 return value after calling the function. If anything goes wrong you'll get a completely unhelpful exception from trying to access status['host'] while status is an integer.