Converting Logstash YAML style configuration file into ElasticSearch Logstash JSON style config

Question

I am new in python and have written first code to convert Logstash YAML style configuration file into ElasticSearch Logstash JSON style config. Please have a look at my piece of code and let me know how can i make it better.

Type of input file:

input:
  file:
     type: "logRecords"
     start_position: "beginning"
     path: "location"
     codec:
         multiline:
           pattern: "regex"
           negate: "true"
           what:  "previous"

filter:
  if:
   type: 'logrecord'
   plugins:
   - grok:
      match:
       message: 'regex1'
      patterns_dir: 'dir1'
   - grok:
      match:
       message: 'regex2'
      patterns_dir: 'dir2'
   - aggregate:
        task_id: "%{recordId}"
        code: ''

output:
 file:
  path: 'oplocation'

YAML to Logstash conf file convertor:

import yaml, json, pprint

f = open('../config/test.yml')
dataMap = yaml.safe_load(f)
f.close()

def configInput(content):
    input = []
    dictiterator(content,input)
    return input

def configFilter(content):
    result=[]
    ifplugins = []
    elseplugins = []

    for key, value in content.items():
        # print key,value
        if key=='if':
            ifplugins.append(key+" " +'[type] == \"')
            ifplugins.append(value.get('type') +" \" { ")
            plugins = value.get('plugins')
            createFilterPlugin(plugins,ifplugins)
            ifplugins.append('} \n ')

        elif key =='elif':
            elseplugins.append('else if' + " " + '[type] == \"')
            elseplugins.append(value.get('type') + " \" { ")
            plugins = value.get('plugins')
            createFilterPlugin(plugins,elseplugins)
            elseplugins.append(' } \n ')
        else:
            result.append(key+" { \n ")
            dictiterator(value,result)
            result.append(" } \n")

    result+=ifplugins
    result+=elseplugins
    # result.append("\n } ")
    return result

def createFilterPlugin(plugins,appenders):

    for plugin in plugins:
        for name, pluginValue in plugin.items():
            if name == 'filterCondition':
                for name1, condition in pluginValue.items():
                    if name1 == 'if':
                        appenders+=condtionalCase(condition, name1)
                    elif name1 == 'elif':
                        appenders +=condtionalCase(condition, 'else if')
            else:
                appenders.append(name + "{ ")
                dictiterator(pluginValue,appenders)
                appenders.append( '} ')

def condtionalCase(condition, name1):
        conditionalCase = []
        conditionalCase.append(name1 + ' ( ')
        conditionalCase.append(condition['condition'] +' )')
        # ifplugins1.append(value1.get('condition') + " \" { ")
        conditionalPlugins = condition.get('plugins')
        for items in conditionalPlugins:
            dictiterator(items,conditionalCase)
            # conditionalCase+=plugin
        # print conditionalCase
        return conditionalCase

def configoutput(content):
    output = []
    for k,v in content.iteritems():
            output.append(k +" { \n ")
            for k,item in v.iteritems():
                if isinstance(item,list):
                    output.append(k+" => "+ ', '.join(item))
                # print k +"=>" +' , '.join(item)
                else:
                    output.append(k+" => "+ item)
            output.append(" } ")
    return output
def dictiterator(input,plugin):

    for k, v in input.iteritems():
        if isinstance(v, dict):
            plugin.append(k + '{ \n ')
            dictiterator(v,plugin)
            plugin.append( '} \n ')
        elif isinstance(v,list):
            # print v
            for item in v:
                if isinstance(item,dict):
                    dictiterator(item,plugin)
                # elif isinstance(item,str):
                #     print ite =m
        else:
            plugin.append( "{0} => {1}".format(k, v))
            plugin.append(" \n ")
    # return plugin

for k, v in dataMap.items():

        inputs = []
        filterplugin = []
        output = []
        finalOp = []

        if k == 'input':
            inputs.append('input { \n ')
            input = configInput(v)
            inputs += input
            inputs.append(' } \n ')
            # inputs = input + " ".join(input)+" } \n "

        elif k =='filter':
            filterplugin.append('filter { \n ')
            filters  = configFilter(v)
            filterplugin+= filters
            filterplugin.append(' }  \n')
            # op = filter+" ".join(filters)+" } \n "
            # print op
        elif k=='output':
           output.append(" output { \n ")
           temp = configoutput(v)
           # print temp
           output+= temp
           output.append(' } \n ')


        # print ' '.join(inputs)+op+''.join(output)
        finalOp+=inputs
        finalOp+=filterplugin
        finalOp+=output
        print ''.join(finalOp)

Answer 1

• While it would be nice to see PEP8 formatting at least consistent formatting is a requirement. That includes spaces around operators and such. Also the indentation is off in places, perhaps that's caused by copy & pasting it here though.
• Commented out code should usually be deleted. That's what a VCS is for in case you need to refer back to it later on.
• Use with on closeable things to make sure resources are released even in case of exceptions.
• There are already a lot of functions, which is good, but there's still some global code. In general it's preferable to put everything into functions (and have a main function) which makes working in the REPL and from other modules much easier.
• Just initialise lists directly instead of calling append on a known empty list.
• Use the function form of print to be consistent and forward compatible.
• You can probably always use iteritems as you already did in a few places.
• All the temporary variables clutter up the code. If they're immediately assigned, appended, ..., elsewhere there's no reason to have them at all.

Otherwise looks good; I'd consider doing an even more function style and not passing in the lists, but just appending the result of the function calls, e.g. ... += dictiterator(...). That way the code can be even more compact as you can just concatenate and return lists instead of having to worry about the order of the append and += calls.

Looks like this know, but can still be worked on given the above remarks:

import yaml

def configInput(content):
    input = []
    dictiterator(content,input)
    return input

def configFilter(content):
    result = []
    ifplugins = []
    elseplugins = []

    for key, value in content.iteritems():
        if key == 'if':
            ifplugins.append(key + ' [type] == \"')
            ifplugins.append(value.get('type') +" \" { ")
            createFilterPlugin(value.get('plugins'), ifplugins)
            ifplugins.append('} \n ')
        elif key == 'elif':
            elseplugins.append('else if [type] == \"')
            elseplugins.append(value.get('type') + " \" { ")
            createFilterPlugin(value.get('plugins'), elseplugins)
            elseplugins.append(' } \n ')
        else:
            result.append(key + " { \n ")
            dictiterator(value, result)
            result.append(" } \n")

    return result + ifplugins + elseplugins

def createFilterPlugin(plugins, appenders):
    for plugin in plugins:
        for name, pluginValue in plugin.iteritems():
            if name == 'filterCondition':
                for name1, condition in pluginValue.iteritems():
                    if name1 == 'if':
                        appenders += conditionalCase(condition, name1)
                    elif name1 == 'elif':
                        appenders += conditionalCase(condition, 'else if')
            else:
                appenders.append(name + "{ ")
                dictiterator(pluginValue, appenders)
                appenders.append( '} ')

def conditionalCase(condition, name1):
    conditionalCase = [name1 + ' ( ', condition['condition'] +' )']
    for items in condition.get('plugins'):
        dictiterator(items, conditionalCase)
    return conditionalCase

def listify(x):
    return x if isinstance(x, list) else [x]

def configoutput(content):
    output = []
    for k, v in content.iteritems():
        output.append([k + " { \n "] +
                       [k + " => " + ', '.join(listify(item))
                        for k, item in v.iteritems()] +
                      [" } "])
    return output

def dictiterator(input, plugin):
    for k, v in input.iteritems():
        if isinstance(v, dict):
            plugin.append(k + '{ \n ')
            dictiterator(v, plugin)
            plugin.append('} \n ')
        elif isinstance(v, list):
            for item in v:
                if isinstance(item, dict):
                    dictiterator(item, plugin)
        else:
            plugin += ["{0} => {1}".format(k, v), " \n "]

def main(filename):
    with open(filename) as f:
        dataMap = yaml.safe_load(f)

    for k, v in dataMap.iteritems():
        inputs = []
        filterplugin = []
        output = []

        if k == 'input':
            inputs += ['input { \n '] + configInput(v) + [' } \n ']
        elif k == 'filter':
            filterplugin += ['filter { \n '] + configFilter(v) + [' }  \n']
        elif k == 'output':
            output += [" output { \n "] + configoutput(v) + [' } \n ']

        print(''.join(inputs + filterplugin + output))

if __name__ == "__main__":
    main('../config/test.yml')