8
\$\begingroup\$

I have a 60GB file of domain names. Some of them are repeated several times and some of them have sub domains that need stripping out too.

For example this might give you a better idea of what the file looks like.

email.example.com
test.staging.lol.uni.ac.uk
hello.test.com
email.example.com
test.com

After parsing the file I would like to create another file that contains.

example.com
uni.ac.uk
test.com

As you can see duplicate base domains are removed and we've left with just the base domain and the top level domain with no duplicates.

Here is my attempt so far and while it works, it's dog slow.

    string[] topLevelDomains = new string[] { ".travelersinsurance", ".accountants", ".lamborghini", ".progressive", ".productions", ".theguardian", ".blackfriday", ".engineering", ".enterprises", ".photography", ".investments", ".motorcycles", ".blockbuster", ".contractors", ".rightathome", ".schaeffler", ".foundation", ".swiftcover", ".apartments", ".associates", ".supersport", ".immobilien", ".industries", ".healthcare", ".africa.com", ".technology", ".eurovision", ".nationwide", ".consulting", ".restaurant", ".republican", ".accountant", ".protection", ".properties", ".university", ".directory", ".community", ".education", ".montblanc", ".accenture", ".microsoft", ".melbourne", ".marshalls", ".marketing", ".financial", ".lifestyle", ".frontdoor", ".lancaster", ".solutions", ".barcelona", ".insurance", ".institute", ".analytics", ".police.uk", ".homegoods", ".amsterdam", ".security", ".partners", ".computer", ".services", ".telecity", ".lighting", ".showtime", ".football", ".clothing", ".pharmacy", ".training", ".cleaning", ".attorney", ".budapest", ".observer", ".holdings", ".brussels", ".supplies", ".broadway", ".guardian", ".graphics", ".download", ".capetown", ".discover", ".discount", ".uconnect", ".diamonds", ".ventures", ".business", ".builders", ".democrat", ".pictures", ".feedback", ".plumbing", ".exchange", ".mortgage", ".esurance", ".software", ".engineer", ".catering", ".property", ".delivery", ".genting", ".gallery", ".hosting", ".hoteles", ".holiday", ".hangout", ".hamburg", ".guitars", ".watches", ".weather", ".website", ".wedding", ".whoswho", ".windows", ".winners", ".youtube", ".zuerich", ".flowers", ".florist", ".flights", ".fitness", ".fishing", ".finance", ".markets", ".fashion", ".farmers", ".express", ".exposed", ".network", ".domains", ".cruises", ".organic", ".courses", ".coupons", ".country", ".cooking", ".contact", ".company", ".cologne", ".college", ".citadel", ".pioneer", ".caravan", ".capital", ".recipes", ".brother", ".rentals", ".booking", ".science", ".singles", ".surgery", ".systems", ".theatre", ".tickets", ".trading", ".academy", ".kitchen", ".com.vc", ".aip.ee", ".org.ee", ".org.uz", ".net.uz", ".com.uz", ".net.lr", ".fie.ee", ".org.uy", ".net.uy", ".mil.uy", ".gub.uy", ".edu.uy", ".com.uy", ".com.eg", ".nsn.us", ".med.ee", ".isa.us", ".fed.us", ".dni.us", ".edu.eg", ".org.lr", ".plc.uk", ".org.uk", ".nhs.uk", ".net.uk", ".gov.lr", ".ltd.uk", ".gov.uk", ".edu.lr", ".com.lr", ".eun.eg", ".org.ug", ".com.ug", ".edu.bh", ".com.ar", ".com.bh", ".grp.lk", ".gov.bf", ".ltd.lk", ".gov.eg", ".web.lk", ".soc.lk", ".com.vi", ".k12.vi", ".net.vi", ".org.vi", ".lib.ee", ".com.vn", ".ngo.lk", ".net.vn", ".org.vn", ".edu.vn", ".edu.lk", ".gov.vn", ".org.lk", ".int.vn", ".com.lk", ".int.lk", ".biz.vn", ".net.lk", ".sch.lk", ".pro.vn", ".com.vu", ".edu.vu", ".gov.lk", ".net.vu", ".org.vu", ".gov.ee", ".edu.ee", ".cn.com", ".com.ws", ".gov.lc", ".net.ws", ".org.ws", ".edu.lc", ".org.lc", ".gov.ws", ".mar.it", ".net.lc", ".edu.ws", ".com.lc", ".photos", ".mil.ec", ".gob.ec", ".org.lb", ".alt.za", ".net.lb", ".edu.za", ".gov.za", ".gov.lb", ".law.za", ".edu.lb", ".com.lb", ".mil.za", ".net.za", ".ngo.za", ".nis.za", ".org.hu", ".org.la", ".nom.za", ".com.la", ".per.la", ".gov.la", ".org.za", ".edu.la", ".web.za", ".gov.ec", ".edu.ec", ".abarth", ".org.ua", ".net.ua", ".physio", ".gov.ua", ".edu.ua", ".com.ua", ".mil.eg", ".net.la", ".int.la", ".piaget", ".com.kz", ".org.ec", ".mil.tz", ".mil.kz", ".abbott", ".abbvie", ".gov.kz", ".net.kz", ".edu.kz", ".pro.ec", ".med.ec", ".k12.ec", ".active", ".idv.tw", ".org.tw", ".net.tw", ".com.tw", ".mil.tw", ".gov.tw", ".edu.tw", ".net.eg", ".org.eg", ".edu.tt", ".gov.tt", ".org.kz", ".fin.ec", ".net.ec", ".com.de", ".net.ky", ".com.ec", ".africa", ".int.tt", ".pro.tt", ".agency", ".biz.tt", ".net.tt", ".org.tt", ".com.tt", ".org.ky", ".sci.eg", ".travel", ".art.dz", ".com.ky", ".kep.tr", ".edu.tr", ".k12.tr", ".mil.tr", ".pol.tr", ".bel.tr", ".gov.tr", ".tel.tr", ".pol.dz", ".bbs.tr", ".gov.ky", ".edu.ky", ".com.se", ".gen.tr", ".web.tr", ".org.tr", ".net.tr", ".biz.tr", ".airbus", ".com.tr", ".com.es", ".nom.es", ".mil.to", ".edu.to", ".org.to", ".net.to", ".gov.to", ".com.to", ".org.es", ".airtel", ".alipay", ".edu.dz", ".alsace", ".rnu.tn", ".rns.tn", ".alstom", ".gov.dz", ".pictet", ".de.com", ".anquan", ".org.tn", ".net.tn", ".nat.tn", ".net.dz", ".ind.tn", ".gov.tn", ".fin.tn", ".ens.tn", ".com.tn", ".gob.es", ".edu.tm", ".mil.tm", ".gov.tm", ".nom.tm", ".net.tm", ".org.tm", ".org.bb", ".com.tm", ".edu.es", ".gov.tl", ".com.et", ".gov.et", ".web.tj", ".org.dz", ".org.tj", ".nic.tj", ".net.tj", ".net.bb", ".mil.tj", ".int.tj", ".gov.tj", ".lom.it", ".edu.tj", ".com.tj", ".eu.com", ".biz.tj", ".gov.bb", ".org.et", ".edu.bb", ".net.th", ".com.bb", ".lig.it", ".gb.com", ".laz.it", ".fvg.it", ".edu.et", ".biz.et", ".aramco", ".emr.it", ".net.et", ".com.dz", ".org.sz", ".biz.bb", ".gb.net", ".com.fr", ".org.sy", ".com.sy", ".mil.sy", ".net.sy", ".gov.sy", ".edu.sy", ".web.do", ".gov.sx", ".nom.fr", ".red.sv", ".org.sv", ".gob.sv", ".edu.sv", ".com.sv", ".prd.fr", ".sld.do", ".org.do", ".hu.com", ".author", ".net.do", ".hu.net", ".mil.do", ".spb.su", ".gov.do", ".cam.it", ".gob.do", ".nov.su", ".edu.do", ".com.do", ".art.do", ".bayern", ".gov.dm", ".edu.dm", ".org.dm", ".net.dm", ".com.dm", ".jp.net", ".cal.it", ".berlin", ".pro.cy", ".bharti", ".mil.kr", ".org.cy", ".net.cy", ".kr.com", ".bas.it", ".blanco", ".ltd.cy", ".gov.cy", ".com.cy", ".org.st", ".net.st", ".mil.st", ".gov.st", ".biz.cy", ".edu.st", ".no.com", ".com.st", ".qc.com", ".ru.com", ".abr.it", ".org.so", ".net.so", ".com.so", ".gov.cx", ".org.cw", ".net.cw", ".org.sn", ".edu.it", ".edu.sn", ".com.sn", ".art.sn", ".tra.kp", ".bostik", ".org.sl", ".gov.sl", ".edu.sl", ".net.sl", ".com.sl", ".cci.fr", ".edu.cw", ".broker", ".rep.kp", ".mil.sh", ".org.sh", ".gov.sh", ".net.sh", ".com.sh", ".com.cw", ".per.sg", ".edu.sg", ".gov.sg", ".org.sg", ".net.sg", ".com.sg", ".inf.cu", ".gov.cu", ".net.cu", ".org.cu", ".edu.cu", ".com.cu", ".org.kp", ".camera", ".gov.kp", ".com.ge", ".edu.kp", ".com.kp", ".com.ba", ".edu.ge", ".org.se", ".gov.ge", ".gov.kn", ".org.ge", ".mil.ge", ".edu.kn", ".net.ge", ".career", ".org.kn", ".net.kn", ".pvt.ge", ".gov.it", ".net.gg", ".org.gg", ".fhv.se", ".quebec", ".sa.com", ".com.gh", ".edu.gh", ".gov.gh", ".org.gh", ".casino", ".racing", ".mil.gh", ".mil.ba", ".com.gi", ".ltd.gi", ".gov.ba", ".gov.sd", ".realty", ".med.sd", ".edu.sd", ".org.sd", ".net.sd", ".com.sd", ".gov.gi", ".edu.sc", ".org.sc", ".net.sc", ".gov.sc", ".com.sc", ".mod.gi", ".org.sb", ".net.sb", ".gov.sb", ".edu.sb", ".com.sb", ".edu.gi", ".sch.sa", ".edu.sa", ".pub.sa", ".med.sa", ".gov.sa", ".org.sa", ".net.sa", ".com.sa", ".org.gi", ".web.co", ".mil.rw", ".int.rw", ".edu.ba", ".com.rw", ".int.is", ".edu.rw", ".net.rw", ".gov.rw", ".com.km", ".rec.co", ".mil.ru", ".gov.ru", ".org.co", ".nom.co", ".center", ".snz.ru", ".net.co", ".mil.co", ".int.co", ".chanel", ".nkz.ru", ".ass.km", ".gov.co", ".mil.km", ".edu.km", ".chrome", ".church", ".kms.ru", ".circle", ".org.is", ".cmw.ru", ".prd.km", ".edu.co", ".claims", ".gov.km", ".clinic", ".nom.km", ".com.co", ".org.km", ".vrn.ru", ".gov.is", ".coffee", ".comsec", ".condos", ".coupon", ".credit", ".com.ki", ".reisen", ".udm.ru", ".gov.ki", ".org.ki", ".net.ki", ".biz.ki", ".tsk.ru", ".edu.ki", ".net.ba", ".tom.ru", ".mil.kg", ".dating", ".datsun", ".stv.ru", ".gov.kg", ".spb.ru", ".edu.kg", ".com.kg", ".net.kg", ".dealer", ".org.kg", ".degree", ".rnd.ru", ".ptz.ru", ".org.ba", ".edu.is", ".dental", ".edu.an", ".design", ".com.is", ".nsk.ru", ".net.is", ".nov.ru", ".org.an", ".direct", ".net.an", ".msk.ru", ".se.com", ".com.an", ".org.al", ".biz.az", ".repair", ".doosan", ".report", ".mil.jo", ".gov.jo", ".sch.jo", ".edu.jo", ".net.jo", ".dunlop", ".org.jo", ".khv.ru", ".dupont", ".durban", ".com.jo", ".pro.az", ".org.je", ".net.je", ".sch.ir", ".review", ".emerck", ".energy", ".jar.ru", ".org.ir", ".net.ir", ".gov.ie", ".net.al", ".estate", ".gov.ir", ".events", ".expert", ".mil.az", ".us.org", ".mil.al", ".family", ".cbg.ru", ".rocher", ".web.id", ".bir.ru", ".mil.cn", ".org.cn", ".se.net", ".net.cn", ".gov.cn", ".edu.cn", ".rogers", ".viajes", ".org.ru", ".net.ru", ".int.ru", ".edu.ru", ".com.ru", ".net.iq", ".com.gl", ".gov.al", ".gov.rs", ".edu.az", ".edu.rs", ".org.rs", ".viking", ".edu.gl", ".www.ro", ".com.cn", ".org.az", ".net.cm", ".rec.ro", ".flickr", ".nom.ro", ".villas", ".edu.al", ".org.ro", ".com.ro", ".net.gl", ".nom.re", ".gov.cm", ".com.re", ".org.gl", ".sch.qa", ".org.qa", ".net.qa", ".com.cm", ".mil.qa", ".gov.qa", ".edu.qa", ".com.qa", ".gov.az", ".org.py", ".net.py", ".mil.py", ".gov.py", ".edu.py", ".com.al", ".com.py", ".com.gn", ".mil.cl", ".int.az", ".net.az", ".ryukyu", ".com.az", ".safety", ".edu.gn", ".virgin", ".com.pt", ".org.ai", ".int.pt", ".edu.pt", ".org.pt", ".gov.pt", ".net.pt", ".gov.gn", ".net.ps", ".org.ps", ".com.ps", ".plo.ps", ".sec.ps", ".gov.ps", ".edu.ps", ".org.gn", ".sakura", ".gob.cl", ".gov.cl", ".futbol", ".vision", ".org.iq", ".gallup", ".net.gn", ".com.aw", ".net.ai", ".est.pr", ".int.ci", ".garden", ".biz.pr", ".pro.pr", ".sanofi", ".edu.pr", ".gov.pr", ".org.pr", ".net.pr", ".com.pr", ".com.gp", ".net.gp", ".net.pn", ".edu.pn", ".org.pn", ".school", ".gov.pn", ".net.ci", ".edu.gp", ".com.iq", ".george", ".schule", ".edu.ci", ".mil.iq", ".edu.iq", ".giving", ".gov.iq", ".com.ai", ".uk.com", ".global", ".com.io", ".waw.pl", ".off.ai", ".com.ci", ".voting", ".org.ci", ".gov.cd", ".vic.au", ".nom.ag", ".google", ".uk.net", ".tas.au", ".eu.int", ".gratis", ".qld.au", ".us.com", ".voyage", ".uy.com", ".mil.in", ".vuelos", ".gov.in", ".walter", ".nsw.au", ".res.in", ".health", ".warman", ".edu.in", ".hermes", ".shouji", ".hiphop", ".act.au", ".sch.id", ".hockey", ".nic.in", ".ind.in", ".webcam", ".gov.bz", ".asn.au", ".edu.bz", ".org.bz", ".net.bz", ".hughes", ".gov.au", ".com.bz", ".gen.in", ".com.by", ".mil.by", ".gov.by", ".org.bw", ".net.ag", ".org.in", ".imamat", ".net.in", ".org.bt", ".net.bt", ".gov.bt", ".insure", ".org.ag", ".intuit", ".edu.ac", ".edu.bt", ".com.bt", ".gov.bs", ".edu.bs", ".jaguar", ".org.bs", ".net.bs", ".com.bs", ".zlg.br", ".net.id", ".vet.br", ".com.ag", ".tur.br", ".joburg", ".trd.br", ".tmp.br", ".teo.br", ".juegos", ".kaufen", ".srv.br", ".slg.br", ".rec.br", ".kinder", ".kindle", ".edu.af", ".qsl.br", ".psi.br", ".psc.br", ".pro.br", ".kyknet", ".za.com", ".elk.pl", ".lancia", ".ppg.br", ".org.br", ".latino", ".odo.br", ".lawyer", ".ntr.br", ".net.af", ".lefrak", ".biz.id", ".not.br", ".edu.au", ".net.br", ".mus.br", ".soccer", ".mil.br", ".org.af", ".med.br", ".com.af", ".mat.br", ".social", ".lel.br", ".living", ".org.au", ".leg.br", ".gov.af", ".locker", ".net.au", ".jus.br", ".jor.br", ".london", ".mil.id", ".gr.com", ".inf.br", ".ind.br", ".imb.br", ".gov.br", ".org.im", ".ggf.br", ".luxury", ".com.au", ".madrid", ".g12.br", ".maison", ".makeup", ".fst.br", ".net.im", ".market", ".mattel", ".fot.br", ".fnd.br", ".ven.it", ".vda.it", ".far.br", ".eti.br", ".etc.br", ".esp.br", ".vao.it", ".eng.br", ".emp.br", ".edu.br", ".eco.br", ".ecn.br", ".gov.pl", ".com.ac", ".mobily", ".cnt.br", ".mil.ae", ".cng.br", ".cim.br", ".sos.pl", ".bmd.br", ".monash", ".sex.pl", ".rel.pl", ".gov.ae", ".in.net", ".mormon", ".com.im", ".nom.pl", ".moscow", ".mil.pl", ".bio.br", ".xihuan", ".ato.br", ".art.br", ".gsm.pl", ".arq.br", ".edu.pl", ".biz.pl", ".gov.as", ".atm.pl", ".mutual", ".aid.pl", ".org.pl", ".net.pl", ".com.pl", ".org.gp", ".sch.ae", ".gos.pk", ".gop.pk", ".gon.pk", ".gok.pk", ".gob.pk", ".gov.pk", ".web.pk", ".biz.pk", ".fam.pk", ".org.pk", ".edu.pk", ".net.pk", ".com.pk", ".nagoya", ".com.gr", ".mil.ph", ".ngo.ph", ".edu.ph", ".gov.ph", ".org.ph", ".net.ph", ".com.ph", ".edu.gr", ".edu.pf", ".org.pf", ".com.pf", ".net.gr", ".net.pe", ".com.pe", ".org.pe", ".mil.pe", ".nom.pe", ".gob.pe", ".edu.pe", ".org.gr", ".nom.pa", ".med.pa", ".abo.pa", ".ing.pa", ".net.pa", ".edu.pa", ".sld.pa", ".org.pa", ".com.pa", ".gob.pa", ".xperia", ".gov.gr", ".com.gt", ".pro.om", ".org.om", ".net.om", ".natura", ".med.om", ".gov.om", ".edu.om", ".com.om", ".studio", ".edu.gt", ".agr.br", ".adv.br", ".org.nz", ".net.nz", ".adm.br", ".mil.nz", ".org.ae", ".mil.bo", ".iwi.nz", ".net.bo", ".yachts", ".gen.nz", ".org.bo", ".cri.nz", ".com.br", ".gob.gt", ".ind.gt", ".com.nr", ".net.nr", ".org.nr", ".edu.nr", ".gov.nr", ".int.bo", ".biz.nr", ".mil.gt", ".net.gt", ".net.ae", ".org.gt", ".gob.bo", ".mil.ng", ".gov.ng", ".sch.ng", ".org.ng", ".net.ng", ".gov.bo", ".edu.ng", ".com.ng", ".co.com", ".edu.bo", ".umb.it", ".nom.ad", ".nissan", ".supply", ".web.nf", ".rec.nf", ".per.nf", ".net.nf", ".com.nf", ".com.gy", ".net.gy", ".com.hk", ".edu.hk", ".gov.hk", ".org.na", ".com.na", ".com.bo", ".sydney", ".tos.it", ".taa.it", ".target", ".org.ac", ".mil.ac", ".pro.na", ".org.bm", ".idv.hk", ".net.bm", ".mil.my", ".edu.my", ".gov.my", ".org.my", ".net.my", ".com.my", ".net.hk", ".net.mx", ".edu.mx", ".gob.mx", ".org.mx", ".com.mx", ".org.hk", ".org.mw", ".net.mw", ".gov.bm", ".int.mw", ".gov.mw", ".edu.mw", ".edu.bm", ".com.mw", ".tattoo", ".biz.mw", ".tur.ar", ".com.hn", ".pro.mv", ".org.mv", ".net.mv", ".com.bm", ".office", ".mil.mv", ".int.mv", ".olayan", ".gov.mv", ".edu.mv", ".org.ar", ".com.mv", ".biz.mv", ".org.bi", ".edu.hn", ".museum", ".sic.it", ".net.ac", ".sex.hu", ".gov.mu", ".org.mu", ".net.mu", ".com.mu", ".org.hn", ".org.mt", ".net.mt", ".edu.mt", ".com.mt", ".net.hn", ".org.ms", ".net.ms", ".gov.ms", ".edu.ms", ".com.ms", ".mil.hn", ".gov.mr", ".gob.hn", ".net.ar", ".mil.ar", ".online", ".gov.mo", ".edu.mo", ".org.mo", ".net.mo", ".com.mo", ".com.hr", ".org.mn", ".edu.mn", ".gov.mn", ".com.ht", ".edu.bi", ".org.ml", ".net.ml", ".gov.ml", ".com.bi", ".edu.ml", ".com.ml", ".oracle", ".orange", ".inf.mk", ".gov.mk", ".edu.mk", ".net.mk", ".org.mk", ".com.mk", ".tennis", ".otsuka", ".int.ar", ".gov.ar", ".com.mg", ".mil.mg", ".edu.mg", ".gob.ar", ".prd.mg", ".gov.mg", ".nom.mg", ".org.mg", ".net.ht", ".gov.bh", ".its.me", ".gov.me", ".sar.it", ".edu.me", ".org.me", ".net.me", ".gov.ac", ".pro.ht", ".org.ht", ".org.bh", ".tienda", ".med.ht", ".pug.it", ".edu.ar", ".org.ma", ".gov.ma", ".net.ma", ".pmn.it", ".art.ht", ".ae.org", ".org.ly", ".med.ly", ".sch.ly", ".edu.ly", ".plc.ly", ".gov.ly", ".net.ly", ".com.ly", ".net.bh", ".ar.com", ".asn.lv", ".net.lv", ".br.com", ".mil.lv", ".org.lv", ".gov.lv", ".edu.lv", ".com.lv", ".pol.ht", ".mol.it", ".gov.lt", ".edu.ht", ".org.ls", ".msk.su", ".total", ".ge.it", ".tours", ".fr.it", ".pb.ao", ".co.ao", ".fm.it", ".trade", ".trust", ".fi.it", ".fg.it", ".og.ao", ".gv.ao", ".fe.it", ".fc.it", ".ed.ao", ".en.it", ".tunes", ".cz.it", ".ct.it", ".cs.it", ".cr.it", ".co.it", ".cn.it", ".cl.it", ".ci.it", ".ch.it", ".vegas", ".ce.it", ".cb.it", ".video", ".vista", ".ca.it", ".co.uz", ".co.vi", ".me.uk", ".co.uk", ".ac.uk", ".ne.ug", ".go.ug", ".sc.ug", ".ac.ug", ".or.ug", ".co.ug", ".zt.ua", ".zp.ua", ".vn.ua", ".uz.ua", ".te.ua", ".sm.ua", ".ac.vn", ".sb.ua", ".rv.ua", ".pl.ua", ".od.ua", ".mk.ua", ".lv.ua", ".lt.ua", ".lg.ua", ".kv.ua", ".ks.ua", ".kr.ua", ".km.ua", ".ac.za", ".co.za", ".kh.ua", ".if.ua", ".dp.ua", ".dn.ua", ".cv.ua", ".cr.ua", ".cn.ua", ".ck.ua", ".tm.za", ".in.ua", ".tv.tz", ".sc.tz", ".or.tz", ".ne.tz", ".me.tz", ".go.tz", ".co.tz", ".ac.tz", ".actor", ".adult", ".aetna", ".co.tt", ".nc.tr", ".dr.tr", ".av.tr", ".tv.tr", ".co.tm", ".go.tj", ".co.tj", ".ac.tj", ".or.th", ".mi.th", ".in.th", ".go.th", ".co.th", ".ac.th", ".archi", ".ac.sz", ".co.sz", ".audio", ".autos", ".azure", ".baidu", ".beats", ".tm.cy", ".bible", ".bingo", ".black", ".boats", ".co.st", ".tm.fr", ".ac.cy", ".boots", ".bosch", ".build", ".tm.se", ".sa.cr", ".pp.se", ".or.cr", ".cards", ".go.cr", ".fi.cr", ".ed.cr", ".co.gg", ".co.cr", ".fh.se", ".bd.se", ".ac.se", ".ac.cr", ".tv.sd", ".co.rw", ".ac.rw", ".co.gl", ".chase", ".cheap", ".chloe", ".cisco", ".citic", ".click", ".cloud", ".coach", ".codes", ".crown", ".tw.cn", ".mo.cn", ".cymru", ".hk.cn", ".dabur", ".zj.cn", ".dance", ".yn.cn", ".xz.cn", ".xj.cn", ".tj.cn", ".sx.cn", ".deals", ".sn.cn", ".sh.cn", ".sd.cn", ".sc.cn", ".qh.cn", ".nx.cn", ".nm.cn", ".ln.cn", ".dodge", ".jx.cn", ".js.cn", ".jl.cn", ".drive", ".hn.cn", ".hl.cn", ".dubai", ".hi.cn", ".he.cn", ".hb.cn", ".ha.cn", ".earth", ".gx.cn", ".edeka", ".email", ".epost", ".epson", ".gz.cn", ".gs.cn", ".gd.cn", ".fj.cn", ".cq.cn", ".faith", ".bj.cn", ".ah.cn", ".fedex", ".final", ".pp.ru", ".ac.ru", ".in.rs", ".ac.rs", ".co.rs", ".ac.cn", ".nt.ro", ".tm.ro", ".ac.gn", ".co.cm", ".go.pw", ".ed.pw", ".or.pw", ".ne.pw", ".co.pw", ".forex", ".forum", ".co.cl", ".md.ci", ".gallo", ".ac.pr", ".games", ".go.ci", ".co.pn", ".ac.ci", ".ed.ci", ".gifts", ".gives", ".glade", ".glass", ".co.ci", ".globo", ".gmail", ".or.ci", ".gc.ca", ".yk.ca", ".sk.ca", ".qc.ca", ".pe.ca", ".green", ".gripe", ".group", ".gucci", ".on.ca", ".guide", ".nu.ca", ".nt.ca", ".ns.ca", ".nl.ca", ".nf.ca", ".nb.ca", ".mb.ca", ".bc.ca", ".ab.ca", ".homes", ".horse", ".house", ".of.by", ".iinet", ".ikano", ".co.bw", ".intel", ".irish", ".jetzt", ".tv.br", ".koeln", ".kyoto", ".av.it", ".at.it", ".weber", ".weibo", ".ar.it", ".aq.it", ".ap.it", ".ao.it", ".an.it", ".al.it", ".works", ".ag.it", ".world", ".ac.ae", ".xerox", ".yahoo", ".co.ae", ".zippo", ".id.ir", ".co.ir", ".ac.ir", ".ac.in", ".za.bz", ".co.in", ".tv.im", ".tt.im", ".co.im", ".ac.im", ".or.id", ".my.id", ".go.id", ".co.id", ".ac.id", ".co.ca", ".co.nl", ".nokia", ".co.na", ".ws.na", ".tv.na", ".cc.na", ".in.na", ".ca.na", ".mx.na", ".us.na", ".dr.na", ".or.na", ".nowtv", ".co.mw", ".ac.mw", ".omega", ".or.mu", ".co.mu", ".ac.mu", ".iz.hr", ".or.bi", ".osaka", ".co.bi", ".co.mg", ".tm.mg", ".ac.me", ".co.me", ".tm.mc", ".paris", ".ac.ma", ".co.ma", ".id.ly", ".parts", ".id.lv", ".party", ".co.ls", ".ac.lk", ".photo", ".co.hu", ".co.lc", ".ac.be", ".tm.hu", ".tv.bb", ".pizza", ".place", ".poker", ".co.bb", ".praxi", ".press", ".prime", ".rs.ba", ".sc.kr", ".re.kr", ".pe.kr", ".or.kr", ".ne.kr", ".ms.kr", ".kg.kr", ".hs.kr", ".go.kr", ".es.kr", ".co.kr", ".ac.kr", ".promo", ".co.ba", ".quest", ".rehab", ".tm.km", ".reise", ".or.jp", ".ne.jp", ".lg.jp", ".gr.jp", ".go.jp", ".ed.jp", ".co.jp", ".ad.jp", ".ac.jp", ".co.je", ".vv.it", ".vt.it", ".vs.it", ".vr.it", ".ricoh", ".vi.it", ".pp.az", ".rocks", ".rodeo", ".ve.it", ".vc.it", ".vb.it", ".va.it", ".ud.it", ".tv.it", ".ts.it", ".tr.it", ".tp.it", ".to.it", ".tn.it", ".salon", ".te.it", ".ta.it", ".sv.it", ".wa.au", ".ss.it", ".sr.it", ".sp.it", ".sener", ".so.it", ".seven", ".si.it", ".sa.au", ".sa.it", ".nt.au", ".ro.it", ".rn.it", ".rm.it", ".shoes", ".ri.it", ".rg.it", ".oz.au", ".id.au", ".re.it", ".rc.it", ".ra.it", ".pz.it", ".pv.it", ".pu.it", ".pt.it", ".skype", ".pr.it", ".sling", ".smart", ".po.it", ".pn.it", ".smile", ".pi.it", ".pg.it", ".solar", ".pe.it", ".pd.it", ".pc.it", ".or.at", ".space", ".gv.at", ".co.at", ".pa.it", ".ot.it", ".ac.at", ".or.it", ".stada", ".store", ".og.it", ".nu.it", ".study", ".no.it", ".style", ".sucks", ".na.it", ".mt.it", ".ms.it", ".swiss", ".mo.it", ".mn.it", ".tatar", ".mi.it", ".me.it", ".mc.it", ".mb.it", ".lu.it", ".lt.it", ".lo.it", ".li.it", ".tires", ".tirol", ".le.it", ".lc.it", ".tmall", ".kr.it", ".today", ".is.it", ".tokyo", ".im.it", ".tools", ".gr.it", ".it.ao", ".go.it", ".toray", ".legal", ".lexus", ".mp.br", ".lilly", ".linde", ".lipsy", ".loans", ".locus", ".lotte", ".lotto", ".lupin", ".macys", ".mango", ".fm.br", ".media", ".miami", ".tm.pl", ".money", ".mopar", ".pc.pl", ".movie", ".am.br", ".nadex", ".ac.pa", ".co.om", ".tv.bo", ".nexus", ".co.nz", ".ac.nz", ".bv.nl", ".co.gy", ".nikon", ".ninja", ".bz.it", ".bt.it", ".bs.it", ".vodka", ".br.it", ".bo.it", ".bn.it", ".bl.it", ".bi.it", ".bg.it", ".wales", ".co.ag", ".ba.it", ".watch", ".co.ve", ".lease", ".prod", ".prof", ".able", ".surf", ".adac", ".yoga", ".talk", ".raid", ".ally", ".read", ".army", ".work", ".auto", ".reit", ".kiwi", ".baby", ".band", ".bank", ".taxi", ".land", ".rent", ".rest", ".info", ".rich", ".beer", ".best", ".life", ".like", ".jobs", ".bike", ".bing", ".limo", ".blog", ".blue", ".link", ".live", ".loan", ".bofa", ".loft", ".zero", ".bond", ".love", ".aero", ".team", ".luxe", ".buzz", ".room", ".cafe", ".rsvp", ".call", ".camp", ".tech", ".care", ".cars", ".casa", ".cash", ".meet", ".meme", ".mobi", ".menu", ".cbre", ".safe", ".mini", ".mint", ".cern", ".co.no", ".chat", ".city", ".club", ".cool", ".name", ".sale", ".tips", ".date", ".deal", ".moto", ".diet", ".dish", ".docs", ".b.br", ".save", ".duck", ".duns", ".post", ".navy", ".vote", ".fail", ".town", ".fans", ".farm", ".fast", ".fiat", ".film", ".fire", ".fish", ".toys", ".news", ".next", ".scot", ".ford", ".a.se", ".seat", ".b.se", ".c.se", ".d.se", ".e.se", ".f.se", ".seek", ".g.se", ".h.se", ".i.se", ".fund", ".nico", ".k.se", ".l.se", ".m.se", ".n.se", ".o.se", ".p.se", ".r.se", ".nike", ".zone", ".game", ".tube", ".s.se", ".t.se", ".u.se", ".w.se", ".x.se", ".y.se", ".z.se", ".sexy", ".gent", ".gift", ".show", ".gold", ".silk", ".site", ".golf", ".skin", ".xbox", ".goog", ".sohu", ".open", ".guge", ".guru", ".song", ".help", ".here", ".sony", ".weir", ".page", ".host", ".pars", ".spot", ".c.la", ".wiki", ".pics", ".imdb", ".zara", ".asia", ".immo", ".star", ".ping", ".pink", ".play", ".plus", ".wine", ".java", ".porn", ".gap", ".sap", ".run", ".fly", ".rio", ".fan", ".ren", ".esq", ".qvc", ".dwg", ".pru", ".dot", ".pin", ".dnp", ".pet", ".dev", ".ott", ".day", ".net", ".csc", ".ong", ".cfd", ".one", ".ceo", ".obi", ".cbs", ".ntt", ".edu", ".nra", ".xxx", ".now", ".abb", ".ngo", ".aco", ".new", ".aeg", ".nba", ".cba", ".mtn", ".anz", ".mov", ".car", ".moi", ".axa", ".gov", ".zip", ".mma", ".bbt", ".mlb", ".bcn", ".meo", ".bzh", ".med", ".bot", ".mba", ".bet", ".ltd", ".bio", ".lol", ".bmw", ".law", ".cal", ".yun", ".you", ".xyz", ".int", ".xin", ".wtf", ".wtc", ".wme", ".win", ".wed", ".vip", ".vet", ".ups", ".trv", ".top", ".thd", ".tci", ".tax", ".tab", ".stc", ".srt", ".srl", ".soy", ".sky", ".ski", ".sex", ".tel", ".mil", ".sas", ".foo", ".rip", ".fit", ".ril", ".eus", ".red", ".eat", ".pub", ".pnc", ".dog", ".pid", ".ooo", ".dad", ".onl", ".crs", ".biz", ".off", ".ceb", ".nyc", ".cbn", ".nrw", ".com", ".org", ".aaa", ".nhk", ".abc", ".nfl", ".ads", ".nec", ".msd", ".app", ".mom", ".bar", ".pro", ".bbc", ".mit", ".cab", ".men", ".buy", ".man", ".bid", ".bnl", ".lat", ".joy", ".jnj", ".jmp", ".iwc", ".itv", ".ist", ".ink", ".ing", ".ice", ".how", ".hiv", ".hbo", ".got", ".goo", ".cat", ".gmx", ".bom", ".sew", ".li", ".lc", ".ar", ".aq", ".lb", ".la", ".kz", ".ao", ".ky", ".kr", ".kp", ".kn", ".an", ".am", ".km", ".ki", ".kg", ".jp", ".jo", ".al", ".ai", ".je", ".ag", ".it", ".is", ".af", ".ir", ".iq", ".io", ".in", ".im", ".ae", ".ie", ".ad", ".cx", ".cz", ".de", ".dj", ".cw", ".cv", ".dk", ".dm", ".do", ".cu", ".dz", ".ec", ".cr", ".yt", ".ee", ".ws", ".wf", ".vu", ".vn", ".vi", ".co", ".vg", ".ve", ".vc", ".va", ".uz", ".eg", ".uy", ".us", ".uk", ".ug", ".ua", ".tz", ".tw", ".tv", ".tt", ".es", ".cn", ".tr", ".tp", ".to", ".cm", ".tn", ".tm", ".cl", ".et", ".tl", ".tk", ".tj", ".th", ".tg", ".tf", ".td", ".ci", ".ch", ".cg", ".cf", ".cd", ".cc", ".eu", ".fi", ".tc", ".fm", ".fo", ".fr", ".sz", ".ca", ".sy", ".sx", ".sv", ".bz", ".su", ".st", ".by", ".sr", ".bw", ".bv", ".so", ".sn", ".sm", ".bt", ".sl", ".sk", ".sj", ".si", ".bs", ".sh", ".sg", ".ga", ".gb", ".gd", ".ge", ".gf", ".gg", ".gh", ".gi", ".se", ".sd", ".sc", ".sb", ".sa", ".gl", ".rw", ".ru", ".rs", ".ro", ".re", ".gm", ".gn", ".qa", ".py", ".pw", ".pt", ".ps", ".gp", ".pr", ".pn", ".pm", ".pl", ".pk", ".gq", ".gr", ".ph", ".br", ".pf", ".pe", ".pa", ".gs", ".gt", ".bo", ".om", ".nz", ".nu", ".nr", ".no", ".bm", ".nl", ".gw", ".bj", ".gy", ".ng", ".nf", ".hk", ".ne", ".bi", ".nc", ".bh", ".bg", ".na", ".bf", ".be", ".my", ".mx", ".hm", ".bb", ".hn", ".mw", ".mv", ".mu", ".mt", ".ba", ".ms", ".mr", ".hr", ".mq", ".mp", ".az", ".ax", ".aw", ".mo", ".ht", ".mn", ".ml", ".mk", ".mh", ".mg", ".me", ".md", ".mc", ".ma", ".au", ".ly", ".lv", ".lu", ".at", ".as", ".lt", ".ls", ".lr", ".lk", ".hu", ".id", ".ac" };

    HashSet<string> domainHashList = new HashSet<string>();

    int lineNumber = 0;

    double pc = 0;
    double totalLines = 1906663905;

    var t = Task.Run(() =>
    {
        using (var fileStream = File.OpenRead("F:\\domains-final\\domains\\domains-final.csv"))
        {
            using (var reader = new StreamReader(fileStream))
            {
                while (reader.Peek() >= 0)
                {
                    lineNumber++;

                    pc = Math.Round((lineNumber / totalLines) * 100, 10);

                    Dispatcher.Invoke(() =>
                    {
                        lineLabel.Content = "Line number: " + lineNumber.ToString() + " Percentage: " + pc.ToString();
                    });

                    try
                    {
                        string line = reader.ReadLine();

                        if (!line.Contains('.')) continue;

                        foreach (string topLevelDomain in topLevelDomains)
                        {
                            string domain = line.Trim();

                            if (domain.EndsWith(topLevelDomain))
                            {
                                string cleanedDomainTemp = line.Replace(topLevelDomain, "");

                                if (!cleanedDomainTemp.Contains('.'))
                                {
                                    string cleanedDomain = cleanedDomainTemp + topLevelDomain;

                                    if (domainHashList.Contains(domain)) break;

                                    domainHashList.Add(domain);

                                    File.AppendAllText("F:\\domains-final\\domains\\doms.txt", domain + Environment.NewLine);
                                }
                                else
                                {
                                    string cleanedDomain = cleanedDomainTemp.Split('.').Last() + topLevelDomain;

                                    if (domainHashList.Contains(cleanedDomain)) break;

                                    domainHashList.Add(cleanedDomain);

                                    File.AppendAllText("F:\\domains-final\\domains\\doms.txt", cleanedDomain + Environment.NewLine);
                                }

                                break;
                            }
                        }
                    }
                    catch { }
                }
            }
        }
    });

I'm looking at my computer's resources while running this, and the disk access is at 0-1% and the CPU usage is at 30%, so I'm assuming I'm maxing out just one core.

I have some ideas but I'm not 100% sure how to implement them correctly as C# isn't my main language.

  1. Spawn more threads and use all CPU cores
  2. Attempt to load 1/3 of the file into my 32GB of memory at a time
  3. Buffer up the disk writing and write 1000 domains into the results file at a time
  4. See if I can find a more efficient way of converting to base + TLD (but I can't find one so far)

What improvements can I make to speed this code up?

\$\endgroup\$
0
8
\$\begingroup\$

I would separate out the Reading of lines from a file to it's own method. A nice way to report back progress is the IProgress interface. That way we can just calculate the progress and report it back. You can make this more complex than just the percentage but I just passed back the percentage. if you want line count you can do the code for that. But just beware reporting back to the dispatcher can eat up time. It's why I only push back when the percentage changed. Also stream are going to be buffered so the percentage will be a tad off but with large files it's close enough from my experience.

I personally like the extra brackets on using statements but I also always put brackets on single line if. That to me is a personal preference and you should base it on your coding guidelines.

This method is similar to File.ReadLines but since we needed Progress I created this method.

private static IEnumerable<string> FileReadLines(string file, IProgress<int> progress)
{
    var fileSize = new FileInfo(file).Length;
    using (var fileStream = File.OpenRead(file))
    {
        using (var reader = new StreamReader(fileStream))
        {
            Int? previousProgress = null;
            while (!reader.EndOfStream)
            {
                var line = reader.ReadLine();
                if (line != null)
                {
                    if (progress != null)
                    {
                        var percentDone = (int)Math.Round((reader.BaseStream.Position * 100d) / fileSize, 0);
                        if (previousProgress != percentDone)
                        {
                            progress.Report(percentDone);
                            previousProgress = percentDone;
                        }
                    }
                    yield return line;
                }
            }
        }
    }
}

Now I moved the array of top level domains into a static field and change it to a hashset. Plus I calculated the depth of domains.

private static readonly HashSet<string> _topLevelDomains = new HashSet<string>(new[] {
        ".travelersinsurance", ".accountants", "...."}); // did not include them all here to save ones and zeros
private static int _maxDomainLevel = _topLevelDomains.Max(d => d.Count(x => x == SplitChar));
private const char SplitChar = '.';

Now I created a method to take a line from the file and return back keyvaluepairs based on the potential matches in the top level domain hashset.

private static IEnumerable<KeyValuePair<string, string>> GetDomains(string domain)
{
    var domainParts = domain.Split(SplitChar);
    int start;
    if (domainParts.Length <= _maxDomainLevel)
    {
        start = 1;
    }
    else
    {
        // only need to match on part of the string since we can eliminate any that have more parts than in the top level domain
        start = domainParts.Length - _maxDomainLevel;
    }
    for (var i = start; i < domainParts.Length; i++)
    {
        var range =  domainParts.Length - i;
        // build up the domain from the subparts
        var key = SplitChar + string.Join(SplitChar, Enumerable.Range(i, range)
            .Select(x => domainParts[x]));
        var value = domainParts[i - 1] + key;
        yield return new KeyValuePair<string, string>(key, value);
    }
}

Now that we have all the pieces we can write some PLINQ code to process it all in parallel

public static void ParseFile(string inputfile, string outputFile, IProgress<int> progress)
{
    var domains = FileReadLines(inputfile, progress)
        .AsParallel()
        .Where(x => x.Contains('.'))
        .Select(x => GetDomains(x).FirstOrDefault(kv => _topLevelDomains.Contains(kv.Key)))
        .Select(kv => kv.Value)          
        .Where(x => x != null)
        .Distinct();
    File.AppendAllLines(outputFile, domains);
}

Now you can call it like so. My example I was putting progress to console but you would also push it to the dispatcher.

ParseFile(@"c:\temp\source.txt", @"c:\temp\output.txt", new Progress<int>(l => Console.WriteLine(l)));

I, obviously, don't have a 60Gig file but I believe with using PLINQ and changing the top level domains to be a hash set will be quicker. If you want total control then I would use a producer/consumer like the TPL DataFlow but I think it's over kill for this.

\$\endgroup\$
6
\$\begingroup\$

A 60 GB file fits a lot of domains.

Reporting progress after each of them results in so many calls, that the application may end up spending more time reporting the progress than doing useful work.

Consider reporting the progress less often, say only in 1 or 0.1% increments.

\$\endgroup\$
2
  • 2
    \$\begingroup\$ Lot of domains reminded me of fortune teller men in wagon. I see domains in your future. Jesus that’s a lot of domains :) \$\endgroup\$ – CharlesNRice Dec 25 '19 at 20:24
  • \$\begingroup\$ It is only necessary to update progress when the displayed progress value changes. \$\endgroup\$ – 1201ProgramAlarm Dec 30 '19 at 15:03
3
\$\begingroup\$

When using more than one Using block you actually don't need to nest the second one inside of the first one, so this:

using (var fileStream = File.OpenRead("F:\\domains-final\\domains\\domains-final.csv"))
{
    using (var reader = new StreamReader(fileStream))
    {
...

looks like this instead

using (var fileStream = File.OpenRead("F:\\domains-final\\domains\\domains-final.csv"))
using (var reader = new StreamReader(fileStream))
{
...

inside of your foreach loop I would also invert the first if statement to dump out of the current iteration, this will lessen the amount of indentation that you have inside the foreach loop.

in the if part of the if/else statement inside the foreach loop, you create the variable cleanedDomain but don't use it inside of that statement, once you leave the statement that variable is disposed of, you could remove that assignment as it could save you a little performance but it might not be noticeable. I removed it in the "After" code to clean up some noise.

Foreach loop Before:

foreach (string topLevelDomain in topLevelDomains)
{
    string domain = line.Trim();
    if (domain.EndsWith(topLevelDomain))
    {
        string cleanedDomainTemp = line.Replace(topLevelDomain, "");
        if (!cleanedDomainTemp.Contains('.'))
        {
            string cleanedDomain = cleanedDomainTemp + topLevelDomain;
            if (domainHashList.Contains(domain)) break;

            domainHashList.Add(domain);

            File.AppendAllText("F:\\domains-final\\domains\\doms.txt", domain + Environment.NewLine);
        }
        else
        {
            string cleanedDomain = cleanedDomainTemp.Split('.').Last() + topLevelDomain;

            if (domainHashList.Contains(cleanedDomain)) break;

            domainHashList.Add(cleanedDomain);

            File.AppendAllText("F:\\domains-final\\domains\\doms.txt", cleanedDomain + Environment.NewLine);
        }

        break;
    }
}

Foreach Loop After:

foreach (string topLevelDomain in topLevelDomains)
{
    string domain = line.Trim();
    if (!domain.EndsWith(topLevelDomain)) continue;

    string cleanedDomainTemp = line.Replace(topLevelDomain, "");
    if (!cleanedDomainTemp.Contains('.'))
    {
        if (domainHashList.Contains(domain)) break;

        domainHashList.Add(domain);
        File.AppendAllText("F:\\domains-final\\domains\\doms.txt", domain + Environment.NewLine);
    }
    else
    {
        string cleanedDomain = cleanedDomainTemp.Split('.').Last() + topLevelDomain;
        if (domainHashList.Contains(cleanedDomain)) break;

        domainHashList.Add(cleanedDomain);
        File.AppendAllText("F:\\domains-final\\domains\\doms.txt", cleanedDomain + Environment.NewLine);
    }

    break;
}

I would also caution you against using a try/catch to simply bypass exceptions in your code without handling them appropriately.

You should not catch all exceptions and do nothing in the catch statement, it is bad practice. Instead, you should see what exceptions that do occur and figure out how to handle those situations in your code during the development stage, maybe you catch certain exceptions and perform certain tasks inside that catch statement. Allowing exceptions to bubble up and alert you lets you know that there are issues in your input or issues with a connection to the application, if you simply keep the application running without handling the exception you could be throwing your money into a server and not know it.


Your variables pc and totalLines are magic numbers

it appears that pc is a percentage, but the other variable has what seems to be an arbitrary number 1906663905 you should let yourself know what this number is, or find a way to get it from the file that is being read in (assuming it is the total number of lines in the file that you are reading from.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.