Parsing Solr log files

Question

I am kind of new to programming. Picked up some Perl about a year ago and now learning some Python. I am pretty confident in Perl, but Python seems un-natural to me.

I wrote a little script that parses log files and indexes them into Solr in JSON. This is for some reporting.

I wanted to see if there are more pythonic ways of doing things.

import re
import json
import requests
import argparse
import os
import time

parser = argparse.ArgumentParser(description='')
parser.add_argument('-solr', type=str, nargs='+', help='Address of Solr server (ex: http://192.168.137.128:8983/solr/)')
parser.add_argument('-collection', type=str, nargs='+', help='Name of Collection: (ex: collection1)')
parser.add_argument('-sendinc', type=int, default=1000, nargs=1, help='Number of documents after which to send and commit (ex: 1000)')
parser.add_argument('-commit', type=int, default=10000, nargs=1, help='Number of documents after which to send and commit (ex: 1000)')
parser.add_argument('-logs', type=str, nargs='+', help='Directory of Log Files (ex: /opt/sw/solr/logs/)')
parser.add_argument('-workdir', default='./',type=str, nargs='+', help='Working Directory (ex: /opt/sw/solr/logs/)')
parser.add_argument('-tail',type=int, nargs='?', help='How many seconds to wait to parse files (ex: 60)')
parser.add_argument('-fast',type=int, nargs='?', help='Use for indexing files that are no longer changing. Can\'t Resume ')
parser.add_argument('-noindex',type=int, nargs='?', help='Use for indexing files that are no longer changing. Can\'t Resume ')
args = parser.parse_args()

maindata = ''
maincount = 0
mainparsedfiles = {}
mainparsecontrolfile = ''
currentfile = ''

print(args)
def logtype(filename):
    type = ''
    if "core" in filename and not 'request' in filename:
        type='solrcore'
    return type

def main():
    global mainparsecontrolfile
    readControl()
    if os.path.isfile(args.logs[0]):
        #Process as a file
        parseFile(args.logs[0])
    else:
        #Process Logs Directory
        doDir(args.logs[0])


def readControl():
    global mainparsedfiles
    global mainparsecontrolfile
    if args.workdir[0]:
        if os.path.isfile(args.workdir[0]+'logparser.txt'):
            try:
                mainparsecontrolfile = open(args.workdir[0]+'logparser.txt','r+')
            except:
                print("Couldn't Open Control File for Reading: " + args.workdir[0]+'logparser.txt')
        else:
            try:
                mainparsecontrolfile = open(args.workdir[0]+'logparser.txt','r+')
            except:
                print("Couldn't Create Control File for Reading: " + args.workdir[0]+'logparser.txt')

    #global mainparsecontrolfile
    if mainparsecontrolfile:
        for x in mainparsecontrolfile.readlines():
            a = x[:-1].split('\t')
            mainparsedfiles[a[0]] = a[1]
        mainparsecontrolfile.close()
    return True

def writeControl():
    #global mainparsecontrolfile
    global mainparsedfiles

    if args.workdir[0]:
        if os.path.isfile(args.workdir[0]+'logparser.txt'):
            try:
                mainparsecontrolfile = open(args.workdir[0]+'logparser.txt','w+')
                readControl()
            except:
                print("Couldn't Open Control File: " + args.workdir[0]+'logparser.txt')
        else:
            try:
                mainparsecontrolfile = open(args.workdir[0]+'logparser.txt','w+')
                readControl()
            except:
                print("Couldn't Create Control File for Write: " + args.workdir[0]+'logparser.txt')
    data = ''
    for x in mainparsedfiles:
        data += str(x) + "\t" + str(mainparsedfiles[x]) + "\n"
    mainparsecontrolfile.write(data)


def doDir(directory):
    files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory,f)) and re.search('.log$',f)]
    compressedfiles = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory,f)) and re.search('.log.gz$',f)]
    if files:
        for x in files:
            #print("Parsing " +args.logs[0]+x)
            parseFile(args.logs[0]+x)
        #if not args.fast:
            #doDir(directory)

    if compressedfiles:
        print("Going to Process These Files: ")
        print(compressedfiles)
        for x in compressedfiles:
            os.system("gunzip " + args.logs[0]+x.replace('.gz',''))
            parseFile(args.logs[0]+x)
            os.system("gzip " + args.logs[0]+x)
    else:
        print("No files in the directory that end with .log")


def parseFile(filename):
    global mainparsedfiles
    global currentfile

    comp = 0
    if '.gz' in filename:
        comp = 1
        os.system("gunzip " + filename)
        filename = filename.replace('.gz','')
    currentfile = filename
    count = 0
    type1 = logtype(filename)
    try:
file = open(filename,'r')
    except:
print("Couldn't open File: " + filename)
    pos = 0
    print("Parsing File %s" % (filename))
    if filename in mainparsedfiles:
        pos = file.tell()
        file.seek(0,2)
        if file.tell() == int(mainparsedfiles[filename]):
            print("File is not changed: " + filename + " - " + str(file.tell()))
            file.close()
            return True
        else:
            file.seek(int(mainparsedfiles[filename]),0)

    if type1 == 'solrcore':
        if args.fast:
            print("Wee... Going to go fast")
            for l in file.readlines():
                s = ''
                t = parseSolrCoreLine(l)
                if 'id' in t:
                    count +=1
                    #print ("Indexing Line %s " % (count))#- file pos: %s" % (count, file.tell()))
                    t['id'] = t['id'].replace(':','_')
                    if count % args.sendinc[0] == 0:
                        s = indexToSolr(t,1)
                    else:
                        s = indexToSolr(t,0)
                    if s == False:
                        print("Error at %s" % (count))
            mainparsedfiles[filename] = file.tell()
            writeControl()
            os.system("gzip " + filename)
            mainparsedfiles[filename+'.gz'] = mainparsedfiles[filename]

        else:
            print("Going to Tail")
            startsize = os.path.getsize(filename)
            l = file.readline()
            done = 0
            while l and done != 1:
                s = ''
                #print(l)
                t = parseSolrCoreLine(l)
                if 'id' in t:
                    count +=1
                    print ("Indexing Line %s " % (count))
                    t['id'] = t['id'].replace(':','_')
                    mainparsedfiles[filename] = file.tell()
                    if count % args.commit[0] == 0:
                        s = indexToSolr(t,1)
                        writeControl()
                    else:
                        s = indexToSolr(t,0)
                    if s == False:
                        print("Error at %s" % (count))
                l = file.readline()

                if not l:
                    nowsize = os.path.getsize(filename)
                    if nowsize > startsize:
                        time.sleep(10)
                        l = file.readline()
                    else:
                        done = 1

        t = {}
        t['id'] = 'commit'
        indexToSolr(t,1)
    file.close()
    os.system("gzip " + currentfile)

    os.system("gzip " + filename)
    writeControl()


def writeToFile(data):
    global currentfile
    orig = ''
    if 'json' not in currentfile:
        orig = currentfile
        currentfile = currentfile + ".json"
        if os.path.isfile(currentfile) and not orig in existmainparsedfiles:
            os.system("rm -f " + currentfile)
    output = open(currentfile,'a+')
    output.write(data+'\n')
    output.close()
def filterData(d):
    a = {}
    blocked = ['shard.url','lowercaseOperators','distrib','mm','isShard','f.mmfr_exact.facet.limit','defType','group.ngroups','group','wt','timeAllowed','facet.mincount','fl','boost','facet.threads','ps','f.cat.facet.prefix','qf','group.sort','ids','fq_sas','group.facet','f.attr.facet.limit','group.facet','group.facet''facet']
    for x in d:
        if not x in blocked:
            a[x] = d[x]
    return a

def indexToSolr(f,c):
    f = filterData(f)
    d = json.dumps(f)
    global maincount
    global maindata
    #print ("Main Count is " + str(maincount))
    if c == 1:
        #print("Sending Commit")
        solr = args.solr[0]+args.collection[0]+'/update?commit=true'
    else:
        #print("Sending Data")
        solr = args.solr[0]+args.collection[0]+'/update'

    if maincount == 0:
        maindata = '[\n'
    maincount += 1
    maindata += d + ',\n'
    out = True
    if maincount == args.sendinc[0]:
        maindata = maindata[:-2]
        maindata += ']\n'
        #print(maindata)
        if not args.noindex:
            r = requests.post(solr,data=maindata,headers = {'content-type': 'application/json'})
            if r.status_code != 200:
                print(r.status_code)
                print(r.raw)
                print(r.text)
                out = False
                r.close()

        writeToFile(maindata)
        maindata = ''
        maincount = 0
    return out

def parseSolrCoreLine(l):
#Do string filtering for single, double quotes here. As well as for any unencoded characters
    out = {}
    final = {}
    temp = l.split(' ')
    if len(temp)>3:
        out['date'] = temp[0]
        out['time'] = temp[1]
        out['event_timestamp'] = "%sT%sZ" % (temp[0], temp[1])
        if len(temp)>7 and temp[3] == 'core.SolrCore' and temp[2] == "INFO":
            accepted = ['path','status','QTime','hits']
            out['collection'] = getCollection(temp[5])
            for item in temp[6:len(temp)]:
                #Item is each pair of arguments, ex: status=0
                if re.search('.*=.*',item):
                    #Only proceed if it has an equal sign in it and it is not the params field
                    t2 = item.split('=')
                    for param in t2:
                        if t2[0] in accepted:
                            out[t2[0]] = t2[1].replace('/','')

                if re.search('^params',item):
                    #Parse Params
                    params = parseParams(item)
                    if 'q' in params:
                        out.update(params)
                        out['id'] = out['event_timestamp'] + '_' + out['q']
                    elif 'fq' in out:
                        out.update(params)
                        out['id'] = out['event_timestamp'] + '_' + out['fq']

    return out



def parseParams(d):
    d = d.replace('{','')
    d = d.replace('}','')
    d = d.replace('"','')
    d = d.replace('params=','')
    out = {}
    t= d.split('&')
    count = 0
    for l in t:
        try:
            #l is each set of arguments
            la = l.split('=')
            #la is an array of parameter and value
            if la[0] == 'fq':
                out['fq'] = ''
                lb = la[1].split(':')
                out['fq_'+ lb[0]] = lb[1]
                out['fq'] += lb[0] + ' '
            else:
                if ':' in la[1]:
                    la[1] = la[1].replace(':','=')
                    out[la[0]] = la[1]
                else:
                    out[la[0]] = la[1]
        except:
            print("Problem is Parse Params, dump below:")
            print(l)
            print(out)

    return out



def getCollection(d):
    d = d.replace('[','')
    d = d.replace(']','')
    if re.search('_',d):
        a = re.match('^(.+)_.*_.*',d)
        d = a.group(1)
    return d

def printarray(a):
    for i in range (0,len(a)):
        print( "%r - %r " % (i, a[i]) )
    print("\n\n\n--------\n\n\n")

main()

I think your indentation got borked when you copied the code over. There are a few isolated sections that, if run as is, would error out due to an unexpected indentation block. — BeetDemGuise, Commented Jul 3, 2014 at 14:32

BeetDemGuise · Accepted Answer · 2014-07-03 15:24:15Z

Since this you posted quite a lot of code, I'll keep my comments higher-level and more general in nature.

There is one document titled PEP8 that covers Python's style conventions. Take a look at it as it gives very nice tips and reference points on how to style your Python code.

For basically everything in Python, use underscores_in_names instead of camelCase or other conventions. The only exception is for class names use PascalCase.
Always use with syntax when dealing with files. Using the with keyword, prevents some common errors that can happen when using open and close. The syntax looks like this:
```
with open(filename) as file:
    # Do stuff with the file

# The file pointer is closed at this point.
foo = 'Hello World!'
```
Be specific when using try ... except blocks. This point is two-fold:
- Put only the minimum amount of code inside a try block. This will help isolate any problems that may occur.
- Catch specific errors. Several errors could possibly be thrown from some code. Its always good to know which was thrown.
As a quick note, you never use raise in your except blocks. It seems odd that in a code as extensive as this, no errors will actually cause your program to stop execution.
Don't use global. There really is no practical use case for global, especially one that cannot be implemented using classes or parameters.
When creating strings, using string formatting instead of string concatenation. Even though the performance benefits are unclear, formatting is more Pythonic:
```
# Instead of this...
data += str(x) + "\t" + str(mainparsedfiles[x]) + "\n"

# Do this.
data += '{}\t{}\n'.format(x, mainparsedfiles[x])
```
When creating strings from data in a list, you can use str.join. This function joins all elements in a sequence by the string it was called on. For example, take a for loop from your code:
```
data = ''
for x in mainparsedfiles:
    data += str(x) + "\t" + str(mainparsedfiles[x]) + "\n"
```
This can be written with join as such:
```
data = '\n'.join(['{}\t{}'.format(key, value)
                  for key, value in main_parsed_files.items()])
```

There are other suggestions I can make. However, my review is already getting lengthy, so I'll leave the rest to others.

Thanks man, all good comments. I know I need to re-write a bunch of the script, so I wanted to get some feedback on how to make it more straight forward. — nick_v1, Commented Jul 3, 2014 at 19:25
hey man, I re-wrote most of this, can you give it a quick glance? codereview.stackexchange.com/questions/56214/second-python-code — nick_v1, Commented Jul 5, 2014 at 18:29

Stack Exchange Network

Parsing Solr log files

1 Answer 1

Your Answer

Not the answer you're looking for? Browse other questions tagged
python
beginner
parsing
or ask your own question.

Linked

Hot Network Questions

Parsing Solr log files

1 Answer 1

Your Answer

Sign up or log in

Post as a guest

Not the answer you're looking for? Browse other questions tagged pythonbeginnerparsing or ask your own question.

Linked

Related

Hot Network Questions

Not the answer you're looking for? Browse other questions tagged
python
beginner
parsing
or ask your own question.