I've written a little piece of code to filter Firewall-Connection-Events by a certain group of IPs. But the code can't keep up since the input is fairly huge. I am looking for ways to make this code more efficient in order to handle the Firewall-Connection-Events.
[READ_CLIENTS] The code starts with an empty list (client_IPs). The list is getting filled with unique IPs (read_clients), if the size of the file changes "read_clients" is called again to alter the list.
[READ_EVENTS] To get the events, I've used While True to loop over the event-File and return the events with yield - if there is no new input, then sleep for 0.1 seconds.
[PROCESS_AND_FILTER] After this I loop over the generator object to compare each event to each Unique-IP and seperate the result in two files.
# -*- coding: utf-8 -*- from IPy import IP import os import time # Create String-Array to compare path_unique_ips = '/var/log/unique.log' # Sophos-UTM Packet-Accepted-Event | Connections to Customer-Net path_sophos_to_customer = '/var/log/packet-accepted-sophos.log' # match logs match_log = '/var/log/matched.log' no_match_log = '/var/log/not_matched.log' # IP-Filter-Array client_IPs =  #get file size of unique ips file size_ip_file = os.stat(path_unique_ips).st_size def read_clients(path_unique_ips): client_IPs_file = open(path_unique_ips, "r") if client_IPs_file.mode == 'r': # read all line from client_IPs_file new_client_IPs = client_IPs_file.readlines() #check for new clients and fill array for new_client_IP in new_client_IPs: if new_client_IP not in client_IPs: client_IPs.append(IP(new_client_IP).strNormal()) def read_events(path_sophos_to_customer): connection_event_to_customer = open(path_sophos_to_customer, 'r') connection_event_to_customer.seek(0, 2) if connection_event_to_customer.mode == 'r': while True: new_event = connection_event_to_customer.readline() if not new_event: time.sleep(0.1) continue yield new_event #file size of unique IP File changed, re-run the function read_clients if size_ip_file is not os.stat(path_unique_ips).st_size: read_clients(path_unique_ips) def process_and_filter(my_events): #get events in generator-object from function read_events # loop over generator-object, filled with events for new_event in my_events: print(new_event) # loop over event with all ips for client in client_IPs: # if client-ip found in event write to match.log and break loop to go for next event if client in new_event: with open(match_log, 'a+') as matched: matched.write(new_event) break # if ip wasn't in event write event to no_match.log else: with open(no_match_log, 'a+') as no_match: no_match.write(new_event) if __name__ == '__main__': read_clients(path_unique_ips) new_events=read_events(path_sophos_to_customer) process_and_filter(new_events)
Jan 18 14:14:14 126.96.36.199 2019: 01:18-14:14:14 firewall-1 ulogd: id="2002" severity="info" sys="SecureNet" sub="packetfilter" name="Packet accepted" action="accept" fwrule="653" initf="eth1" outitf="eth0" srcmac="aa:bb:cc:dd:ee:ff" dstmac="00:11:22:33:44:55" srcip="10.10.10.10" dstip="10.10.10.11" proto="6" length="52" tos="0x00" prec="0x00" ttl="127" srcport="58589" dstport="22" tcpflags="ACK"