I am developing application that goes through 2 websites and gets all the articles, but my code is identical in most parts, is there a way to optimize this code actually :/ (TL and DN are the naming conventions of the news papers)
public class Main {
public static void main(String[] args) {
BlockingQueue<JsonObject> producer2consumer = new LinkedBlockingQueue<JsonObject>();
FakeJsonConsumer consumer = new FakeJsonConsumer(producer2consumer);
consumer.start();
DNCrawler producer = new DNCrawler(producer2consumer);
producer.start();
}
}
public class DNSpecific {
//public static final String[] dnSeeds = { "http://www.dn.se/ekonomi/" }; // Faster testing
public static final String[] dnSeeds = { "http://www.dn.se/",
"http://www.dn.se/ekonomi/",
"http://www.dn.se/sthlm/",
"http://www.dn.se/sport/",
"http://www.dn.se/kultur/",
"http://www.dn.se/ledare/",
"http://www.dn.se/motor/"
};
public static Set<String> getSeedSet() {
HashSet<String> seedSet = new HashSet<String>();
for (String seed : dnSeeds)
seedSet.add(seed);
return seedSet;
}
private static final Pattern DN_FILTER = Pattern.compile(".*((/rss/|/webb-tv/|/fondlistor/|rm=print|/css/|/pages/)).*");
public static boolean isRelevant(String url) {
if (DN_FILTER.matcher(url).matches()) {
//System.out.println("DNDrop "+url);
return false;
}
else if (url.startsWith("/")) {
System.out.println("DN local web reference? "+url);
return false;
}
return url.startsWith("http://www.dn.se/");
}
}
public class DNCrawler extends Thread {
private final int MaxSeedPageRetry = 5;
private final int MaxPageRetry = 20;
private final int IterationSleep = 10; // Time between iterations in minutes
private final int RetrySleep = 500; // Time between retries in milliseconds
private final BlockingQueue<JsonObject> jsonQueue; // Our output channel
public DNCrawler(BlockingQueue<JsonObject> queue) {
super("DNCrawlerThread");
jsonQueue = queue;
System.out.println("Enter "+this.getClass().getName());
}
@Override
public void run() {
System.out.println("DNCrawler up and running ... ");
System.out.println("\n************************************ ");
System.out.println(getTime()+": Initial search started on "+getDay());
// Setup DN specific seeds
Set<String> seedSet = DNSpecific.getSeedSet();
System.out.println("Seeds: " + seedSet);
// Traverse seeds to find reachable
HashMap<String,Document> url2doc = new HashMap<String,Document>();
Set<String> urlSet = traverseSeeds(seedSet,url2doc,MaxSeedPageRetry);
System.out.print("Seed Reachable DN Urls: "+ urlSet.size());
//System.exit(-1);
traverseReachable(urlSet,url2doc,MaxPageRetry);
// Repeat forever
while (true) {
try { Thread.sleep(IterationSleep*60*1000); // Wait for next iteration
} catch (InterruptedException e) { e.printStackTrace();}
System.out.println("\n"+getTime()+": New search started");
urlSet = traverseSeeds(seedSet,url2doc,MaxSeedPageRetry);
System.out.print("Seed Reachable DN Urls: "+ urlSet.size());
// Find new reachable urls
HashSet<String> newReachable = new HashSet<String>();
for (String url : urlSet) {
if (url2doc.get(url) == null) {
//System.out.println("New: "+url);
newReachable.add(url);
}
}
// Fetch new reachable pages
traverseReachable(newReachable,url2doc,MaxPageRetry);
}
}
private Set<String> traverseSeeds(Set<String> seedSet, HashMap<String,Document> url2doc, int maxRetry) {
ArrayDeque<String> toVisit = new ArrayDeque<String>(seedSet);
HashSet<String> urlSet = new HashSet<String>();
int maxAttempts = maxRetry + toVisit.size();
int tryCount = 0;
while (!toVisit.isEmpty() && tryCount<maxAttempts) {
tryCount++;
String seedUrl = toVisit.removeFirst();
Document seedPage = null;
try {
seedPage = Jsoup.connect(seedUrl).get();
url2doc.put(seedUrl, seedPage);
// Find all outgoing links and add relevant links
Elements links = seedPage.select("a[href]");
for (Element link : links) {
String url = link.attr("abs:href").trim().toLowerCase();
if ( !seedSet.contains(url) && isRelevant(url)) {
urlSet.add(url);
}
}
}
catch (SocketTimeoutException ex) { // Time-out ==> add to queue agqin and sleep for a while
toVisit.add(seedUrl);
try { Thread.sleep(RetrySleep); // Calm down!
} catch (InterruptedException e) { e.printStackTrace();}
}
catch (Exception ex) { // HTML fetch problem ==> drop (in this iteration)
System.err.println("\t"+ex.getMessage()+" "+seedUrl);
}
}
if (tryCount == maxAttempts)
System.err.println("Failed to download all seeds within given limit ("+maxRetry+") of retries!");
return urlSet;
}
private void traverseReachable(Set<String> reachableSet, HashMap<String,Document> url2doc, int maxRetry) {
System.out.print(", New Reachable Urls: "+ reachableSet.size());
long startTime = System.currentTimeMillis();
ArrayDeque<String> toVisit = new ArrayDeque<String>(reachableSet);
int maxAttempts = maxRetry + toVisit.size();
int tryCount = 0, errorCount = 0, newCount = 0;
while (!toVisit.isEmpty() && tryCount<maxAttempts) {
tryCount++;
String url = toVisit.removeFirst();
Document page = null;
try {
page = Jsoup.connect(url).get();
url2doc.put(url, page);
if (saveIfArticle(url,page) )
newCount++;
}
catch (SocketTimeoutException ex) {
//System.out.println(tryCount+"\tTIME-OUT: "+ url);
toVisit.add(url);
try { Thread.sleep(RetrySleep); // Calm down!
} catch (InterruptedException e) { e.printStackTrace();}
}
catch (Exception ex) {
//System.err.println(tryCount+"\t"+ex.getMessage()+" "+url);
errorCount++;
}
}
if (tryCount == maxAttempts)
System.err.println("Failed to download all reachable pages within given limit ("+maxRetry+") of retries!");
else {
System.out.print(", New Articles: "+newCount);
System.out.print(", Errors: "+errorCount+", Required retries: "+(tryCount-reachableSet.size()));
long ellapsedTime = System.currentTimeMillis() - startTime;
System.out.println(", Reachable traversal done in "+(ellapsedTime/1000)+" seconds");
}
}
/*
* Article identification and extraction
*/
private boolean saveIfArticle(String url, Document page) {
// article tag ==> an article in DN
Elements articleElements = page.getElementsByTag("article");
if (!articleElements.isEmpty()) {
buildArticle(url,page);
return true;
}
return false;
}
/* Url filtering
* - Remove unwanted file/image references
* - Apply DN specific url filtering
*
*/
private static final Pattern FILE_FILTER = Pattern.compile(
".*(\\.(css|js|bmp|gif|jpe?g|png|tiff?|mid|mp2|mp3|mp4|wav|avi|mov|mpeg|ram|m4v|pdf" +
"|rm|smil|wmv|swf|wma|zip|rar|gz))$");
private boolean isRelevant(String url) {
if (url.length() < 1) // Remove empty urls
return false;
else if (FILE_FILTER.matcher(url).matches()) { // Ignore urls matching our defined set of unwanted image/file extensions.
//System.out.println("GeneralDrop "+url);
return false;
}
else
return DNSpecific.isRelevant(url);
}
/*
* Extract text and build Article/Json object,
*/
private void buildArticle(String url, Document page) {
long discoveryTime = System.currentTimeMillis();
String asText = page.toString();
try {
ArticleTextExtractor extractor = new ArticleTextExtractor();
JResult res = extractor.extractContent(asText);
String title = res.getTitle();
String text = res.getText();
//System.out.println(title);
System.out.println(text);
Article a = new Article();
a.setWebsite("http://www.dn.se/");
a.setUrl(url);
a.setTitle(title);
a.setDiscoveryTime(discoveryTime);
a.setText(text);
//System.out.println(a);
// Add to queue ==> forward article to consumer
try { jsonQueue.put(a); }
catch (InterruptedException ex) { ex.printStackTrace();}
}
catch (Exception e) {
e.printStackTrace();
}
//System.exit(-1);
}
/*
* Utility methods
*
*/
SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss");
private String getTime() {
Date date = new Date();
String dateTime = timeFormat.format( date);
return dateTime;
}
SimpleDateFormat dayFormat = new SimpleDateFormat("yyyy-MM-dd");
private String getDay() {
Date date = new Date();
String dateTime = dayFormat.format( date);
return dateTime;
}
}
And here is the code for the second robot:
public class Main {
public static void main(String[] args) {
BlockingQueue<JsonObject> producer2consumer = new LinkedBlockingQueue<JsonObject>();
FakeJsonConsumer consumer = new FakeJsonConsumer(producer2consumer);
consumer.start();
TLCrawler producer = new TLCrawler(producer2consumer);
producer.start();
}
}
public class TLSpecific {
//public static final String[] dnSeeds = { "http://www.dn.se/ekonomi/" }; // Faster testing
public static final String[] tlSeeds = { "http://www.thelocal.se/",
"http://www.thelocal.se/page/view/national/",
"http://www.thelocal.se/page/view/money/",
"http://www.thelocal.se/page/view/politics/",
"http://www.thelocal.se/page/view/society/",
"http://www.thelocal.se/page/view/scitech/",
"http://www.thelocal.se/page/view/education/",
"http://www.thelocal.se/page/view/sport/",
"http://www.thelocal.se/page/view/analysis/",
"http://www.thelocal.se/page/view/features/",
"http://www.thelocal.se/page/view/businesstravel2015/",
"http://www.thelocal.se/page/view/study-at-malmo-university/"
};
public static Set<String> getSeedSet() {
HashSet<String> seedSet = new HashSet<String>();
for (String seed : tlSeeds)
seedSet.add(seed);
return seedSet;
}
private static final Pattern DN_FILTER = Pattern.compile(".*((/rss/|/webb-tv/|/fondlistor/|rm=print|/css/|/pages/)).*");
public static boolean isRelevant(String url) {
if (DN_FILTER.matcher(url).matches()) {
//System.out.println("DNDrop "+url);
return false;
}
else if (url.startsWith("/")) {
System.out.println("TheLocal local web reference? "+url);
return false;
}
return url.startsWith("http://www.thelocal.se/");
}
}
public class TLCrawler extends Thread {
private final int MaxSeedPageRetry = 5;
private final int MaxPageRetry = 20;
private final int IterationSleep = 10; // Time between iterations in minutes
private final int RetrySleep = 500; // Time between retries in milliseconds
private final BlockingQueue<JsonObject> jsonQueue; // Our output channel
public TLCrawler(BlockingQueue<JsonObject> queue) {
super("TLCrawlerThread");
jsonQueue = queue;
System.out.println("Enter "+this.getClass().getName());
}
@Override
public void run() {
System.out.println("TLCrawler up and running ... ");
System.out.println("\n************************************ ");
System.out.println(getTime()+": Initial search started on "+getDay());
// Setup DN specific seeds
Set<String> seedSet = TLSpecific.getSeedSet();
System.out.println("Seeds: " + seedSet);
// Traverse seeds to find reachable
HashMap<String,Document> url2doc = new HashMap<String,Document>();
Set<String> urlSet = traverseSeeds(seedSet,url2doc,MaxSeedPageRetry);
System.out.print("Seed Reachable TL Urls: "+ urlSet.size());
//System.exit(-1);
traverseReachable(urlSet,url2doc,MaxPageRetry);
// Repeat forever
while (true) {
try { Thread.sleep(IterationSleep*60*1000); // Wait for next iteration
} catch (InterruptedException e) { e.printStackTrace();}
System.out.println("\n"+getTime()+": New search started");
urlSet = traverseSeeds(seedSet,url2doc,MaxSeedPageRetry);
System.out.print("Seed Reachable TL Urls: "+ urlSet.size());
// Find new reachable urls
HashSet<String> newReachable = new HashSet<String>();
for (String url : urlSet) {
if (url2doc.get(url) == null) {
newReachable.add(url);
}
}
// Fetch new reachable pages
traverseReachable(newReachable,url2doc,MaxPageRetry);
}
}
private Set<String> traverseSeeds(Set<String> seedSet, HashMap<String,Document> url2doc, int maxRetry) {
ArrayDeque<String> toVisit = new ArrayDeque<String>(seedSet);
HashSet<String> urlSet = new HashSet<String>();
int maxAttempts = maxRetry + toVisit.size();
int tryCount = 0;
while (!toVisit.isEmpty() && tryCount<maxAttempts) {
tryCount++;
String seedUrl = toVisit.removeFirst();
Document seedPage = null;
try {
seedPage = Jsoup.connect(seedUrl).get();
url2doc.put(seedUrl, seedPage);
// Find all outgoing links and add relevant links
Elements links = seedPage.select("a[href]");
for (Element link : links) {
String url = link.attr("abs:href").trim().toLowerCase();
if ( !seedSet.contains(url) && isRelevant(url)) {
urlSet.add(url);
}
}
}
catch (SocketTimeoutException ex) { // Time-out ==> add to queue agqin and sleep for a while
toVisit.add(seedUrl);
try { Thread.sleep(RetrySleep); // Calm down!
} catch (InterruptedException e) { e.printStackTrace();}
}
catch (Exception ex) { // HTML fetch problem ==> drop (in this iteration)
System.err.println("\t"+ex.getMessage()+" "+seedUrl);
}
}
if (tryCount == maxAttempts)
System.err.println("Failed to download all seeds within given limit ("+maxRetry+") of retries!");
return urlSet;
}
private void traverseReachable(Set<String> reachableSet, HashMap<String,Document> url2doc, int maxRetry) {
System.out.print(", New Reachable Urls: "+ reachableSet.size());
long startTime = System.currentTimeMillis();
ArrayDeque<String> toVisit = new ArrayDeque<String>(reachableSet);
int maxAttempts = maxRetry + toVisit.size();
int tryCount = 0, errorCount = 0, newCount = 0;
while (!toVisit.isEmpty() && tryCount<maxAttempts) {
tryCount++;
String url = toVisit.removeFirst();
Document page = null;
try {
page = Jsoup.connect(url).get();
url2doc.put(url, page);
if (saveIfArticle(url,page) )
newCount++;
}
catch (SocketTimeoutException ex) {
//System.out.println(tryCount+"\tTIME-OUT: "+ url);
toVisit.add(url);
try { Thread.sleep(RetrySleep); // Calm down!
} catch (InterruptedException e) { e.printStackTrace();}
}
catch (Exception ex) {
//System.err.println(tryCount+"\t"+ex.getMessage()+" "+url);
errorCount++;
}
}
if (tryCount == maxAttempts)
System.err.println("Failed to download all reachable pages within given limit ("+maxRetry+") of retries!");
else {
System.out.print(", New Articles: "+newCount);
System.out.print(", Errors: "+errorCount+", Required retries: "+(tryCount-reachableSet.size()));
long ellapsedTime = System.currentTimeMillis() - startTime;
System.out.println(", Reachable traversal done in "+(ellapsedTime/1000)+" seconds");
}
}
private boolean saveIfArticle(String url, Document page) {
// article tag ==> an article in DN
Elements articleElements = page.getElementsByTag("article");
if (!articleElements.isEmpty()) {
buildArticle(url,page);
return true;
}
return false;
}
private static final Pattern FILE_FILTER = Pattern.compile(
".*(\\.(css|js|bmp|gif|jpe?g|png|tiff?|mid|mp2|mp3|mp4|wav|avi|mov|mpeg|ram|m4v|pdf" +
"|rm|smil|wmv|swf|wma|zip|rar|gz))$");
private boolean isRelevant(String url) {
if (url.length() < 1) // Remove empty urls
return false;
else if (FILE_FILTER.matcher(url).matches()) {
return false;
}
else
return TLSpecific.isRelevant(url);
}
/*
* Extract text and build Article/Json object,
*/
private void buildArticle(String url, Document page) {
long discoveryTime = System.currentTimeMillis();
String asText = page.toString();
try {
ArticleTextExtractor extractor = new ArticleTextExtractor();
JResult res = extractor.extractContent(asText);
String title = res.getTitle();
String text = res.getText();
//System.out.println(title);
// System.out.println(text);
Article a = new Article();
a.setWebsite("http://www.thelocal.se/");
a.setUrl(url);
a.setTitle(title);
a.setDiscoveryTime(discoveryTime);
a.setText(text);
// System.out.println(a.toJsonString());
//System.out.println(a);
// Add to queue ==> forward article to consumer
try { jsonQueue.put(a); }
catch (InterruptedException ex) { ex.printStackTrace();}
}
catch (Exception e) {
e.printStackTrace();
}
//System.exit(-1);
}
SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss");
private String getTime() {
Date date = new Date();
String dateTime = timeFormat.format( date);
return dateTime;
}
SimpleDateFormat dayFormat = new SimpleDateFormat("yyyy-MM-dd");
private String getDay() {
Date date = new Date();
String dateTime = dayFormat.format( date);
return dateTime;
}
}