The idea is that local security/SIEM solutions want to populate their database with information gathered from different security feeds. This is done through importing and monitoring .txt files (for some reason).
The idea behind this, is that I want each feed to have their own function, to easier be able to filter out, or add new feeds. Not to mention that almost every feed has it's own format.
The output of all this, needs to be in the current format, which is:
IP [FEED NAME] DOMAIN [FEED NAME]
I am sure there are better ways, especially to generate the .txt file outputs.
The idea is that the script is to be run by itself, without any input, in a cron file, and if something has to be changed, the script itself should be edited.
All feeds are directly available for everyone, so if you are curious about any formats, the URLs are in the script, with the exception of the first feed (OTX), which needs you to sign up, if needed I can paste a output from this feed.
So, you can copy and paste this script in and run it out of the box, except you might want to comment out get_alienvault.
Questions:
- Due to that every feed has a different format, I am guessing there would be no way for me to have one function that goes through all feeds, or does someone see a way?
- I am sure the output can be heavily modified, at a later date, I was thinking of either outputting to .txt, or use the CSV lib to writerows the same way I do now with the txt files.
- Do I need the main method? Or is there a better way to call all the different functions? Maybe with the possibility to ignore something in the future? Without commenting out the function call?
- The systems that will be importing this are very sensitive to extra whitespaces and so, currently using lstrip/rstrip for this, and
splitlines(). Any better way?
Script:
import requests
import re
from OTXv2 import OTXv2
from collections import defaultdict
from datetime import datetime, timedelta
# Defines the list that is going to populate the different feeds
d = defaultdict(list)
# Your API key for OTX. If you do not have any, please comment out
# "get_alienvault" in the main function.
otx = OTXv2("XXXXX")
class GetFeeds(object):
"""
Main class to gather and output feed results
"""
def download_file(self, url):
"""
Download the feeds specified
:param url: The location of the source to download
:return The content of the request
"""
r = requests.get(url)
return r.content
def ipgrabber(self, results):
"""
:param reults: The results that should be filtered
:return: Only the IP addresses from the object it filtered out
"""
ip = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', results)
return ip
def get_alienvault(self, days):
"""
Gets the information from Alienvault OTX
:param days: How many days the reslts from the feed can be
:return: The IP addresses and domains returned from OTX
"""
mtime = (datetime.now() - timedelta(days=days)).isoformat()
pulses = otx.getsince(mtime)
# Populate the list with the returned results from OTX
for index, feeds in enumerate(pulses):
for pulse in pulses[index]['indicators']:
if pulse['type'] == 'IPv4':
d['alienvaultip'].append(pulse['indicator'])
elif pulse['type'] == 'URL':
d['alienvaulturl'].append(pulse['indicator'])
elif pulse['type'] == 'domain':
d['alienvaultdomain'].append(pulse['indicator'])
return d
def get_malc0de(self):
"""
Gets the information from the Malc0de Feed
:return: The IP addresses returned from Malc0de
"""
url = 'http://malc0de.com/bl/IP_Blacklist.txt'
results = self.download_file(url).splitlines()
for result in results:
if len(result) > 0:
ip = self.ipgrabber(result.lstrip().rstrip())
if ip:
d['malc0deip'].append(result)
return d
def get_talosintel(self):
"""
Gets the information from the TalosIntel Feed
:return: The IP addresses returned from TalosIntel
"""
url = 'http://www.talosintelligence.com/feeds/ip-filter.blf'
results = self.download_file(url).splitlines()
for result in results:
if len(result) > 0:
ip = self.ipgrabber(result.lstrip().rstrip())
if ip:
d['talosintelip'].append(result)
return d
def get_malwaredomainlist(self):
"""
Gets the information from the MDL Feed
:return: The IP addresses and domains returned from MDL
"""
url = 'http://www.malwaredomainlist.com/hostslist/ip.txt'
domains = 'http://www.malwaredomainlist.com/hostslist/hosts.txt'
results = self.download_file(url).splitlines()
results2 = self.download_file(domains).splitlines()
for result in results:
if len(result) > 0:
ip = self.ipgrabber(result.lstrip().rstrip())
if ip:
d['malwaredomainlistip'].append(result)
for result in results2:
result = result.lstrip('127.0.0.1 , ').rstrip()
if not result.startswith('#') \
and not result.startswith('localhost') and len(result) > 0:
d['malwaredomainlistdomain'].append(result)
return d
def get_tornodes(self):
"""
Gets the information from the Torproject overview
:return: The IP addresses of the current TOR exit nodes
"""
url = 'https://check.torproject.org/exit-addresses'
results = self.download_file(url).splitlines()
for result in results:
if len(results) > 0:
ip = self.ipgrabber(result.lstrip().rstrip())
if ip:
d['tornodeip'].append(ip)
return d
def get_ransomwaretracker(self):
"""
Gets the information from the RansomewareTracker Feed
:return: The IP addresses returned from RansomewareTracker
"""
url = 'https://ransomwaretracker.abuse.ch/feeds/csv/'
results = self.download_file(url).splitlines()
for result in results:
if len(result) > 0:
ip = self.ipgrabber(result.lstrip().rstrip())
for singleip in ip:
d['ransomwaretrackerip'].append(singleip)
for result in results:
if re.search(r'\"online\"', result):
for url in re.findall(r',\"\w+\.\w+\"', result):
url = re.sub(',"', '\"', url)
url = re.sub('"', '', url)
d['ransomewaretrackerdomain'].append(url)
return d
def get_bambenek(self):
"""
Gets the information from the RansomewareTracker Feed
:return: The IP addresses and domains returned from Bambenek
"""
url = 'https://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt'
domains = 'http://osint.bambenekconsulting.com/feeds/c2' \
'-dommasterlist.txt'
results = self.download_file(url).splitlines()
results2 = self.download_file(domains).splitlines()
for result in results:
if len(result) > 0:
ip = self.ipgrabber(result.lstrip().rstrip())
if ip:
d['bambenekip'].append(ip)
for result in results2:
if not result.startswith('#') \
and not result.startswith('localhost') and len(result) > 0:
url = result.split(",")[0]
d['bambenekdomain'].append(url)
return d
# Executes the downloading and parsing of feeds
def main(self):
self.get_malc0de()
self.get_alienvault(10)
self.get_talosintel()
self.get_malwaredomainlist()
self.get_tornodes()
self.get_bambenek()
self.get_ransomwaretracker()
# Export IP Addresses to txt file, tagged with their sources
f = open("iplist.txt", "w")
iplist = d['ransomwaretrackerip']
f.write(
"".join("{}\t[Ransomewaretracker]\n".format(t) for t in iplist))
iplist = d['bambenekip']
f.write("".join("{}\t[Bambenek]\n".format(t) for t in iplist))
iplist = d['alienvaultip']
f.write("".join("{}\t[AlienVault]\n".format(t) for t in iplist))
iplist = d['malc0deip']
f.write("".join("{}\t[Malc0de]\n".format(t) for t in iplist))
iplist = d['talosintelip']
f.write("".join("{}\t[TalosIntel]\n".format(t) for t in iplist))
iplist = d['malwaredomainlistip']
f.write("".join("{}\t[MalwareDomainList]\n".format(t) for t in iplist))
f.close()
# Export domains to txt file, tagged with their sources
f = open("domainlist.txt", "w")
domainlist = d['ransomwaretrackerdomain']
f.write("".join(
"{}\t[Ransomewaretracker]\n".format(t) for t in domainlist))
domainlist = d['bambenekdomain']
f.write("".join("{}\t[Bambenek]\n".format(t) for t in domainlist))
domainlist = d['alienvaultdomain']
f.write("".join("{}\t[AlienVault]\n".format(t) for t in domainlist))
domainlist = d['malwaredomainlistdomain']
f.write(
"".join("{}\t[MalwareDomainList]\n".format(t) for t in domainlist))
f.close()
if __name__ == "__main__":
o = GetFeeds()
o.main()
As output was requested, here is one output per feed:
This is one entry for get_alienvault, there will be many arrays of "industries":
[ { u'industries': [ ], u'TLP': u'green', u'description': u'Analysis of HackingTeam Android malware', u'created': u'2016-11-15T03:10:51.515000', u'tags': [ u'android', u'hackingteam', u'hacking team', u'apk', u'mobile', u'spyware' ], u'modified': u'2016-11-15T03:10:51.515000', u'author_name': u'AlienVault', u'public': True, u'references': [ u'http://rednaga.io/2016/11/14/hackingteam_back_for_your_androids/' ], u'targeted_countries': [ ], u'indicators': [ { u'indicator': u'e362a037e70517565d28ab85959e6c9d231b2baf0c2df3b87dfaa1451278e80c', u'description': u'', u'title': u'', u'created': u'2016-11-15T03:10:53', u'content': u'', u'type': u'FileHash-SHA256', u'id': 1398126 }, { u'indicator': u'07278c56973d609caa5f9eb2393d9b1eb41964d24e7e9e7a7e7f9fdfb2bb4c31', u'description': u'', u'title': u'', u'created': u'2016-11-15T03:10:53', u'content': u'', u'type': u'FileHash-SHA256', u'id': 1398127 }, { u'indicator': u'ed33b83be3af715d3fd8ba6ac8b2b551a16697c5a37a9fcebfc40a024cc9b818', u'description': u'', u'title': u'', u'created': u'2016-11-15T03:10:53', u'content': u'', u'type': u'FileHash-SHA256', u'id': 1398128 }, { u'indicator': u'4308286905ee3d3dd1f4171da15050d1672dcc63bfc65e012b18938275e96085', u'description': u'', u'title': u'', u'created': u'2016-11-15T03:10:53', u'content': u'', u'type': u'FileHash-SHA256', u'id': 1398129 }, { u'indicator': u'87efe6a1cbf4d4481c6fa6e2c70a26a0b50a460557a1ad876af9966a571f8a4c', u'description': u'', u'title': u'', u'created': u'2016-11-15T03:10:53', u'content': u'', u'type': u'FileHash-SHA256', u'id': 1398130 }, { u'indicator': u'68.233.237.11', u'description': u'', u'title': u'', u'created': u'2016-11-15T03:10:53', u'content': u'', u'type': u'IPv4', u'id': 1398131 }, { u'indicator': u'66.232.100.221', u'description': u'', u'title': u'', u'created': u'2016-11-15T03:10:53', u'content': u'', u'type': u'IPv4', u'id': 1398132 } ], u'revision': 1.0, u'adversary': u'HackingTeam', u'id': u'582a7cbbc9eef92d31ad7c90', u'name': u'HackingTeam back for your Androids' }, {
Format for get_malc0de Malc0detxt:
// This file will be automatically updated daily and populated with the last 30 days of malicious IP addresses. // Last updated 2016-11-15 103.234.36.75 59.45.79.75 176.103.55.73 108.41.24.140 52.84.125.228 54.230.193.237 54.230.193.52 198.143.138.155 59.188.68.200 222.163.80.69 122.225.98.197 58.218.211.250 61.147.204.48 95.131.28.244 45.79.194.109 193.238.153.44 153.196.238.158 209.90.88.131 94.23.26.102 79.127.127.67 193.238.153.90 189.113.175.249 46.28.67.156 193.238.153.106
Format for get_talosintel Talostxt:
119.29.132.114 1.169.224.242 151.97.190.239 188.247.135.58 216.176.100.240 109.127.8.242 188.219.154.228 216.215.112.149 188.247.135.74 188.247.135.53 188.247.135.99 83.69.233.121 203.170.193.23 109.229.36.65
Format for get_malwaredomainlist: IP addresses MDLtxtfile
103.14.120.121 103.19.89.55 103.224.212.222 103.31.186.207 103.31.186.29 103.4.16.91 103.4.218.22 103.6.196.156 103.8.127.189 103.8.127.205 104.152.215.90 104.245.239.7
Domains: MDLdomainstxt
# MalwareDomainList.com Hosts List # # http://www.malwaredomainlist.com/hostslist/hosts.txt # # Last updated: Fri, 11 Nov 16 06:02:27 +0000 # 127.0.0.1 localhost 127.0.0.1 0koryu0.easter.ne.jp 127.0.0.1 109-204-26-16.netconnexion.managedbroadband.co.uk 127.0.0.1 11.lamarianella.info 127.0.0.1 1866809.securefastserver.com 127.0.0.1 2amsports.com 127.0.0.1 3.bluepointmortgage.com 127.0.0.1 3.coolerpillow.com 127.0.0.1 4.androidislamic.com 127.0.0.1 4.collecorvino.org 127.0.0.1 4.dlevo.com 127.0.0.1 4.e-why.net 127.0.0.1 4.newenergydata.biz 127.0.0.1 4.newenergydata.info 127.0.0.1 4.periziavela.com 127.0.0.1 4.pianetapollo.com 127.0.0.1 4.whereinitaly.com 127.0.0.1 4.whereinlazio.com 127.0.0.1 4.whereinliguria.com
Format for get_tornodes: Tornodes
ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E Published 2016-11-15 07:22:42 LastStatus 2016-11-15 08:03:23 ExitAddress 162.247.72.201 2016-11-15 08:09:55 ExitNode 004FDAC2B5B966F53361FA34D7E6E48D512FE869 Published 2016-11-14 21:58:30 LastStatus 2016-11-14 23:02:24 ExitAddress 103.14.68.50 2016-11-14 23:06:35 ExitNode 006CC1DD17754582618DE2539DAAFE0A96962583 Published 2016-11-14 23:19:25 LastStatus 2016-11-15 00:03:24 ExitAddress 198.50.159.155 2016-11-15 00:03:42 ExitNode 0077BCBA7244DB3E6A5ED2746E86170066684887 Published 2016-11-14 23:28:32
Format for get_ransomewaretracker RWTcsvfile:
############################################################################## # Ransomware Tracker CSV Feed # # Generated on 2016-11-15 15:26:56 UTC # # # # For questions please refer to: # # https://ransomwaretracker.abuse.ch/feeds/ # ############################################################################## # # Firstseen (UTC),Threat,Malware,Host,URL,Status,Registrar,IP address(es),ASN(s),Country "2016-11-15 11:53:22","C2","Locky","185.82.217.29","http://185.82.217.29/message.php","offline","","185.82.217.29","59729","BG" "2016-11-15 10:07:59","Payment Site","TorrentLocker","ojmekzw4mujvqeju.minitili.at","http://ojmekzw4mujvqeju.minitili.at/","online","","95.163.118.67","207027|12695","RU" "2016-11-15 07:39:38","Payment Site","Cerber","ffoqr3ug7m726zou.ndnmuk.top","http://ffoqr3ug7m726zou.ndnmuk.top","offline","Eranet International Limited","","34702","" "2016-11-15 02:48:56","C2","Locky","107.181.174.34","http://107.181.174.34/message.php","offline","","107.181.174.34","46562","US" "2016-11-14 19:15:48","C2","Locky","93.170.123.119","http://93.170.123.119/message.php","offline","","93.170.123.119","48666","CZ" "2016-11-14 11:15:34","Payment Site","Cerber","ffoqr3ug7m726zou.rssh3l.bid","http://ffoqr3ug7m726zou.rssh3l.bid","online","","104.238.215.11","36114","US" "2016-11-14 09:27:25","Payment Site","Cerber","ffoqr3ug7m726zou.2rl0pv.top","http://ffoqr3ug7m726zou.2rl0pv.top","offline","Eranet International Limited","","36114","" "2016-11-14 06:45:30","Payment Site","Cerber","lfdachijzuwx4bc4.vrgdrs.top","http://lfdachijzuwx4bc4.vrgdrs.top","online","Eranet International Limited","104.238.215.11|104.238.215.11|46.22.220.37","36114|34702","US|EE" "2016-11-14 04:11:26","Payment Site","Cerber","ffoqr3ug7m726zou.iocvou.top","http://ffoqr3ug7m726zou.iocvou.top","offline","Eranet International Limited","","36114|34702","" "2016-11-14 01:16:46","Payment Site","Cerber","ahuqfrqk54v3vnzj.8kiec2.top","http://ahuqfrqk54v3vnzj.8kiec2.top","offline","Eranet International Limited","","36114|34702","" "2016-11-14 00:58:48","Payment Site","Cerber","ffoqr3ug7m726zou.bwei9h.top","http://ffoqr3ug7m726zou.bwei9h.top","offline","Erane
Format for get_bambenek banbenek:
############################################################################## # Ransomware Tracker CSV Feed # # Generated on 2016-11-15 15:26:56 UTC # # # # For questions please refer to: # # https://ransomwaretracker.abuse.ch/feeds/ # ############################################################################## # # Firstseen (UTC),Threat,Malware,Host,URL,Status,Registrar,IP address(es),ASN(s),Country "2016-11-15 11:53:22","C2","Locky","185.82.217.29","http://185.82.217.29/message.php","offline","","185.82.217.29","59729","BG" "2016-11-15 10:07:59","Payment Site","TorrentLocker","ojmekzw4mujvqeju.minitili.at","http://ojmekzw4mujvqeju.minitili.at/","online","","95.163.118.67","207027|12695","RU" "2016-11-15 07:39:38","Payment Site","Cerber","ffoqr3ug7m726zou.ndnmuk.top","http://ffoqr3ug7m726zou.ndnmuk.top","offline","Eranet International Limited","","34702","" "2016-11-15 02:48:56","C2","Locky","107.181.174.34","http://107.181.174.34/message.php","offline","","107.181.174.34","46562","US" "2016-11-14 19:15:48","C2","Locky","93.170.123.119","http://93.170.123.119/message.php","offline","","93.170.123.119","48666","CZ" "2016-11-14 11:15:34","Payment Site","Cerber","ffoqr3ug7m726zou.rssh3l.bid","http://ffoqr3ug7m726zou.rssh3l.bid","online","","104.238.215.11","36114","US" "2016-11-14 09:27:25","Payment Site","Cerber","ffoqr3ug7m726zou.2rl0pv.top","http://ffoqr3ug7m726zou.2rl0pv.top","offline","Eranet International Limited","","36114","" "2016-11-14 06:45:30","Payment Site","Cerber","lfdachijzuwx4bc4.vrgdrs.top","http://lfdachijzuwx4bc4.vrgdrs.top","online","Eranet International Limited","104.238.215.11|104.238.215.11|46.22.220.37","36114|34702","US|EE" "2016-11-14 04:11:26","Payment Site","Cerber","ffoqr3ug7m726zou.iocvou.top","http://ffoqr3ug7m726zou.iocvou.top","offline","Eranet International Limited","","36114|34702","" "2016-11-14 01:16:46","Payment Site","Cerber","ahuqfrqk54v3vnzj.8kiec2.top","http://ahuqfrqk54v3vnzj.8kiec2.top","offline","Eranet International Limited","","36114|34702","" "2016-11-14 00:58:48","Payment Site","Cerber","ffoqr3ug7m726zou.bwei9h.top","http://ffoqr3ug7m726zou.bwei9h.top","offline","Erane
Domains: Bambenekdomains
########################################################### ## Master Feed of known, active and non-sinkholed C&Cs domain ## names ## ## Feed generated at: 2016-11-15 15:16 ## ## Feed Provided By: John Bambenek of Bambenek Consulting ## [email protected] // http://bambenekconsulting.com ## Use of this feed is governed by the license here: ## http://osint.bambenekconsulting.com/license.txt ## ## For more information on this feed go to: ## http://osint.bambenekconsulting.com/manual/c2-dommasterlist.txt ## ## All times are in UTC ############################################################# ns1.backdates10.com,Domain used by beebone,2016-11-15 15:09,http://osint.bambenekconsulting.com/manual/beebone.txt ns1.backdates2.com,Domain used by beebone,2016-11-15 15:09,http://osint.bambenekconsulting.com/manual/beebone.txt ns1.backdates3.com,Domain used by beebone,2016-11-15 15:09,http://osint.bambenekconsulting.com/manual/beebone.txt ns1.dnsfor15.com,Domain used by beebone,2016-11-15 15:09,http://osint.bambenekconsulting.com/manual/beebone.txt ns1.dnsfor7.com,Domain used by beebone,2016-11-15 15:09,http://osint.bambenekconsulting.com/manual/beebone.txt ns1.dnsfor8.com,Domain used by beebone,2016-11-15 15:09,http://osint.bambenekconsulting.com/manual/beebone.txt g0spgts27johy2q.ddns.net,Domain used by corebot,2016-11-15 15:08,http://osint.bambenekconsulting.com/manual/corebot.txt k016u4wdspqb10a.ddns.net,Domain used by corebot,2016-11-15 15:08,http://osint.bambenekconsulting.com/manual/corebot.txt o8ehahitsbw0s07.ddns.net,Domain used by corebot,2016-11-15 15:08,http://osint.bambenekconsulting.com/manual/corebot.txt jyalcixnmcjafecuk.net,Domain used by fobber,2016-11-15 15:09,http://osint.bambenekconsulting.com/manual/fobber-iplist.txt suogzzicvq.com,Domain used by fobber,2016-11-15 15:09,http://osint.bambenekconsulting.com/manual/fobber-iplist.txt
