According to the first part: Forum crawler - counts statistics for words in chosen forum topic I take into account the review of Janos and created Iterator for my classes.
This is part of the whole App. I have a problem with dividing it into classes, I'm not sure about this architecture which I've provided.
Descirption:
Topic is the specific topic of the forum
Page is the page of the forum
The aim of this code is to iterate through pages in Topic and create Page for every page.
The code
// example of use:
public class AppMain{
public static void main(String[] args){
String url = "https://4programmers.net/Forum/Newbie/210891-od_czego_zaczac_nauke_programowania";
Topic topic = new Topic(url);
PageIterator it = topic.iterator();
//it.next();
Page page = it.next();
}
}
...
/*
* to consider: what if the view of the post will be changed from 10 to 50?
*/
public class Topic implements Iterable<Page> {
private String urlBase; // URL of the topic on forum
private String pagePattern; // pattern whcich change while changing page number of the topic, for example: ?page=
private PageIteratorEnum pageIteratorType; // type of the forum 4programmers? Codercity? etc.
// to consider: put these variables into abstract class?
public Topic(String url, String pagePattern) {
this.urlBase = url;
this.pagePattern = pagePattern;
}
public Topic(String url) {
this.urlBase = url;
this.pagePattern = "?page=";
}
public void setUrlBase(String html) {
this.urlBase = html;
}
public String getUrlBase() {
return urlBase;
}
public void setPagePattern(String pattern) {
pagePattern = pattern;
}
public String getPagePattern() {
return pagePattern;
}
public String toString() {
return getUrlBase();
}
public void setIterator(PageIteratorEnum type){
pageIteratorType = type;
}
@Override
public PageIterator iterator() {
if(pageIteratorType == null){
throw new RuntimeException("First set iterator type by setIterator() method.");
}
return new PageIteratorFactoryImp1(this).produceIterator(pageIteratorType);
}
}
...
import org.jsoup.nodes.Document;
public class Page {
Document doc;
public Page(Document doc){
this.doc = doc;
}
}
...
import java.util.Iterator;
public interface PageIterator extends Iterator<Page>{
public boolean hasNext();
public Page next();
}
...
public enum PageIteratorEnum {
it4PROGRAMMERS, itCODERCITY;
}
...
public interface PageIteratorFactory {
public PageIterator produceIterator(PageIteratorEnum type);
}
...
public class PageIteratorFactoryImp1 implements PageIteratorFactory{
Topic topic;
public PageIteratorFactoryImp1(Topic topic){
this.topic = topic;
}
public PageIterator produceIterator(PageIteratorEnum type){
if(type.equals("it4PROGRAMMERS")){
return new TopicPageIterator4programmers(topic);
}
return null;
}
}
...
import java.io.IOException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
public class TopicPageIterator4programmers implements PageIterator {
private Topic topic;
private int count = 0;
String urlBase;
String pagePattern;
String browser = "Mozilla/5.0";
NavigatePanel navigation;
public TopicPageIterator4programmers(Topic topic) {
this.topic = topic;
this.urlBase = topic.getUrlBase();
this.pagePattern = topic.getPagePattern();
this.navigation = new NavigatePanel4programmers(urlBase, pagePattern);
}
@Override
public boolean hasNext() {
return ((count + 1) <= navigation.getMaxPageNumber());
}
@Override
public Page next() {
Page result = null;
String newUrl = urlBase + pagePattern + String.valueOf(++count);
Document doc = null;
try {
doc = Jsoup.connect(newUrl).userAgent(browser).get();
result = new Page4programmers(doc, count);
} catch (IOException e) {
System.out.println("something went wrong with downloading the page: " + newUrl);
e.printStackTrace();
}
return result;
}
}
