I've tomcat running on my Macbook pro and a web application. Making a request to a servlet (below i'm showing post method code) and tipping top command in terminal show a java process with 200/300% cpu usage. The servlet make a request to a mongo database to retrieve 389 documents and then iterate over this documents,process the text with a language detector and Stanford ner classifier, and index it to a local solr server. Is it normal? Can i improve code or something else to reduce cpu load?
this is the code
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
SolrServer server = null;
try {
server = new CommonsHttpSolrServer(conf.getString("solr.url"));
} catch (Exception e) {
e.printStackTrace();
}
if (req.getParameter("delete") != null) {
try {
server.deleteByQuery("*:*");
server.commit();
} catch (SolrServerException e) {
e.printStackTrace();
}
}
BasicDBObject findUsersQuery = new BasicDBObject();
findUsersQuery.put("indexed", false);
DBCursor cur = uColl.find(findUsersQuery);
int i = 0;
while (cur.hasNext()) {
Collection<SolrInputDocument> usersProfiles = new ArrayList<SolrInputDocument>();
// String containing comma separated entities like places, organizations
String namedEntities = "";
// String containing other terms associated to facebook user
String bagOfWords = "";
CrowdUser user = Converter.toObject(CrowdUser.class, cur.next());
log.debug("Indexing User:" + user.getFacebook().getFacebookID() + " "
+ user.getFacebook().getFacebookUser().getFirstName() + " "
+ user.getFacebook().getFacebookUser().getLastName());
FacebookUser fu = user.getFacebook().getFacebookUser();
// Hometown
if (fu.getHometown() != null) {
String name = fu.getHometown().getName();
log.debug("Location:" + name);
if (name != null) {
namedEntities += name + " ";
}
}
// Location
if (fu.getLocation() != null) {
String name = fu.getLocation().getName();
log.debug("Hometown:" + name);
if (name != null) {
namedEntities += name + " ";
}
}
// Work
for (FacebookWork work : fu.getWork()) {
log.debug("Works:");
String description = work.getDescription();
String lang = "";
if (description != null)
lang = detector.detectLang(description);
if (description != null && "en".equals(lang)) {
log.debug("Language detected: " + lang);
ArrayList<String> result = NERUtils.getNamedEntities(description,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
if (work.getEmployer() != null && work.getEmployer().getName() != null)
namedEntities += work.getEmployer().getName() + " ";
if (work.getPosition() != null && work.getPosition().getName() != null)
namedEntities += work.getPosition().getName() + " ";
if (work.getLocation() != null && work.getLocation().getName() != null)
namedEntities += work.getLocation().getName() + " ";
}
// Education
for (FacebookEducation education : fu.getEducation()) {
log.debug("Education:");
if (education.getSchool() != null && education.getSchool().getName() != null)
namedEntities += education.getSchool().getName() + " ";
if (education.getDegree() != null && education.getDegree().getName() != null)
namedEntities += education.getSchool().getName() + " ";
if (education.getConcentration() != null) {
for (FacebookDataType concentration : education.getConcentration()) {
namedEntities += concentration.getName() + " ";
}
}
}
String about = fu.getAbout();
if (about != null) {
String[] strings = about.split("\\n");
for (String string : strings) {
log.debug("About: " + string);
String lang = detector.detectLang(string);
log.debug("Language detected: " + lang);
if ("en".equals(lang)) {
ArrayList<String> result = NERUtils.getNamedEntities(string,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
}
}
String bio = fu.getBio();
if (bio != null) {
String[] strings = bio.split("\\n");
for (String string : strings) {
log.debug("Bio: " + string);
String lang = detector.detectLang(string);
log.debug("Language detected: " + lang);
if ("en".equals(lang)) {
ArrayList<String> result = NERUtils.getNamedEntities(string,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
}
}
String quotes = fu.getQuotes();
if (quotes != null) {
String[] strings = quotes.split("\\n");
for (String string : strings) {
log.debug("Quote: " + string);
String lang = detector.detectLang(string);
log.debug("Language detected: " + lang);
if ("en".equals(lang)) {
ArrayList<String> result = NERUtils.getNamedEntities(string,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
}
}
String political = fu.getPolitical();
log.debug("Political : " + political);
if (political != null)
namedEntities += political + " ";
String religion = fu.getReligion();
log.debug("Religion : " + religion);
if (religion != null)
namedEntities += religion + " ";
if (fu.getWebsite() != null) {
String HTMLtext = URLUtils.getURLContent(user.getFacebook().getFacebookUser()
.getWebsite());
if (HTMLtext != null) {
String parsedText = Jsoup.parse(HTMLtext).text();
log.debug("Web Site Content: " + parsedText);
// log.debug("Language detected: " + detector.detectLang(parsedText));
ArrayList<String> result = NERUtils.getNamedEntities(parsedText,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
}
ArrayList<FacebookDocument> pages = (ArrayList<FacebookDocument>) MongoUtils
.getDocumentsByFacebookID(dColl, user.getFacebook().getFacebookUser()
.getFacebookPages_id());
for (FacebookDocument doc : pages) {
FacebookPage page = (FacebookPage) doc;
log.debug("Page: " + page.getFacebook_id() + " " + page.getName());
if (page.getCategory() != null)
bagOfWords += page.getCategory() + " ";
if (page.getDescription() != null) {
String[] strings = page.getDescription().split("\\n");
for (String string : strings) {
log.debug("Page description: " + string);
String lang = detector.detectLang(string);
log.debug("Language detected: " + lang);
if ("en".equals(lang)) {
string = Jsoup.parse(string).text();
ArrayList<String> result = NERUtils.getNamedEntities(string,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
}
}
String name = page.getName();
if (name != null) {
log.debug("Page name: " + name);
String lang = detector.detectLang(name);
log.debug("Language detected: " + lang);
ArrayList<String> result = NERUtils.getNamedEntities(name,
serializedClassifier);
if ("en".equals(lang)) {
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
} else {
namedEntities += result.get(1);
}
}
HashMap<String, String> others = page.getOthers();
for (String key : others.keySet()) {
try {
log.debug("Key: " + key + " Value: " + others.get(key));
if (key.equals("location")) {
try {
ObjectMapper mapper = new ObjectMapper();
JsonNode location = mapper
.readValue(others.get(key), JsonNode.class);
if (location.get("state") != null)
namedEntities += location.get("state").getTextValue() + " ";
if (location.get("country") != null)
namedEntities += location.get("country").getTextValue() + " ";
if (location.get("city") != null)
namedEntities += location.get("city").getTextValue() + " ";
} catch (Exception e) {
log.debug("Parsing Error");
}
} else if (!key.equals("founded") && !key.equals("is_published")
&& !key.equals("hours") && !key.equals("username")) {
String[] strings = others.get(key).split("\\n");
for (String string : strings) {
String lang = detector.detectLang(string);
log.debug("Language detected: " + lang);
if ("en".equals(lang)) {
string = Jsoup.parse(string).text();
ArrayList<String> result = NERUtils.getNamedEntities(string,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
}
}
} catch (NullPointerException e) {
e.printStackTrace();
}
}
}
ArrayList<FacebookDocument> groups = (ArrayList<FacebookDocument>) MongoUtils
.getDocumentsByFacebookID(dColl, user.getFacebook().getFacebookUser()
.getFacebookGroups_id());
for (FacebookDocument doc : groups) {
FacebookGroup group = (FacebookGroup) doc;
log.debug("Group: " + group.getFacebook_id() + " " + group.getName());
if (group.getDescription() != null) {
String[] strings = group.getDescription().split("\\n");
for (String string : strings) {
log.debug("Group description: " + string);
String lang = detector.detectLang(string);
log.debug("Language detected: " + lang);
if ("en".equals(lang)) {
string = Jsoup.parse(string).text();
ArrayList<String> result = NERUtils.getNamedEntities(string,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
}
}
String name = group.getName();
if (name != null) {
log.debug("Group name: " + name);
String lang = detector.detectLang(name);
log.debug("Language detected: " + lang);
ArrayList<String> result = NERUtils.getNamedEntities(name,
serializedClassifier);
if ("en".equals(lang)) {
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
} else {
namedEntities += result.get(1);
}
}
if (group.getDiscussions() != null) {
String discussion = "";
for (FacebookDiscussion disc : group.getDiscussions()) {
String message = disc.getMessage();
log.debug("Discussion: " + message);
if (message != null && "en".equals(detector.detectLang(message))) {
discussion += message + ".\n";
}
if (disc.getComments() != null) {
for (String comment : disc.getComments()) {
log.debug("Comment:" + comment);
if ("en".equals(detector.detectLang(comment))) {
discussion += comment + ".\n";
}
}
}
}
ArrayList<String> result = NERUtils.getNamedEntities(discussion,
serializedClassifier);
namedEntities += result.get(1);
bagOfWords += result.get(0) + " ";
}
}
log.debug("NamedEntities: " + namedEntities);
log.debug("bagOfWords: " + bagOfWords);
SolrInputDocument doc1 = new SolrInputDocument();
doc1.addField("id", user.getFacebook().getFacebookID(), 1.0f);
doc1.addField("name", user.getFacebook().getFacebookUser().getFirstName() + " "
+ user.getFacebook().getFacebookUser().getLastName(), 1.0f);
doc1.addField("bagofwords", bagOfWords, 1.0f);
doc1.addField("namedentities", namedEntities);
usersProfiles.add(doc1);
try {
server.add(usersProfiles);
server.commit();
user.setIndexed(true);
MongoUtils.updateUserData(uColl, user);
} catch (SolrServerException e) {
// TODO Auto-generated catch block
e.printStackTrace();
log.debug("Not committed");
}
i++;
log.debug("Indexed Profile Number:" + i);
}
}
