5
\$\begingroup\$

I'm developing an application that receives and manages e-mails from a server. I manage only XML files and organize, by sender, the e-mails that have XML attachments, while the others are deleted. I create a list of files and create them in the temporary folder. I have a feeling my JavaMail process is slow because it accesses every message. Can I make my process faster?

public static void main(String[] args) {

    try {

        Properties properties = System.getProperties();
        properties.setProperty("mail.store.protocol", "imaps");
        Session session = Session.getDefaultInstance(properties, null);
        Store store = session.getStore("imaps");
        store.connect("imap.gmail.com", "login", "password");

        Folder inbox = store.getDefaultFolder();
        inbox = inbox.getFolder("INBOX");

        inbox.open(Folder.READ_WRITE);

        if (EmailUtil.hasMessage(inbox)) {

            List<Message> listMessages = new ArrayList<>(Arrays.asList(inbox.getMessages()));
            Iterator<Message> iterator = listMessages.iterator();

            while (iterator.hasNext()) {

                Message message = iterator.next();

                if (EmailUtil.hasNoAttachment(message.getContentType())) {
                    EmailUtil.deleteMessage(message);
                    iterator.remove();
                }
            }

            if (EmailUtil.hasMessagesWithAttachment(listMessages)) {

                Set<String> setSender = new HashSet<>();
                List<File> listFiles = new ArrayList<>();

                listMessages = getAttachmentFiles(listMessages, setSender, listFiles);

                if (EmailUtil.hasXmlAttachments(listFiles)) {

                    Set<Folder> setFolder = getFolderSet(store);

                    List<String> listFolderName = getFoldersName(setFolder);

                    removeExistingFoldersInSet(setSender, listFolderName);

                    if (foldersNotExist(setSender)) {
                        createFolder(store, setSender);
                    }

                    moveMessagesToFolders(listMessages, store);
                }
            }
        }

        EmailUtil.closeFolder(inbox);

    } catch (MessagingException | IOException e) {
        e.printStackTrace();
    }
}

private static void removeExistingFoldersInSet(Set<String> setSender, List<String> listFolderName) {
    setSender.removeAll(listFolderName);
}

private static void createFolder(Store store, Set<String> setSenders) throws MessagingException {
    for (String sender : setSenders) {
        store.getFolder(sender).create(Folder.HOLDS_MESSAGES);          
    }
}

private static boolean foldersNotExist(Set<String> setSenders) throws MessagingException {
    return (setSenders.size() > 0);
}

private static Set<Folder> getFolderSet(Store store) throws MessagingException {
    return new HashSet<>(Arrays.asList(store.getDefaultFolder().list()));
}

private static List<String> getFoldersName(Set<Folder> setFolder) throws MessagingException {

    List<String> listFolderName = new ArrayList<>();

    for (Folder folder : setFolder) {
        listFolderName.add(folder.getName());
    }

    return listFolderName;
}

private static List<Message> getAttachmentFiles(List<Message> listMessages, Set<String> senders, List<File> listFile) throws IOException, MessagingException {

    List<Message> listMessagesWithXmlAttachment = new ArrayList<>(listMessages.size());

    for (Message message : listMessages) {

        Multipart multipart = (Multipart) message.getContent();

        for (int i = 0; i < multipart.getCount(); i++) {

            MimeBodyPart mimeBodyPart = (MimeBodyPart) multipart.getBodyPart(i);

            if (Part.ATTACHMENT.equalsIgnoreCase(mimeBodyPart.getDisposition())) {

                if (FileUtil.isXmlFile(mimeBodyPart.getFileName())) {

                    listMessagesWithXmlAttachment.add(message);
                    senders.add(EmailUtil.getSender(message));
                    File file = generateFile(mimeBodyPart);
                    listFile.add(file);

                }
            }
        }
    }

    removeMessagesWithoutXmlFile(listMessages, listMessagesWithXmlAttachment);
    return listMessagesWithXmlAttachment;
}

private static File generateFile(BodyPart bodyPart) throws IOException, MessagingException {

    String path = FileUtil.createFile("", bodyPart.getFileName());
    Files.copy(bodyPart.getInputStream(), Paths.get(path), StandardCopyOption.REPLACE_EXISTING);

    return new File(path);
}

private static void removeMessagesWithoutXmlFile(List<Message> listMessages, List<Message> listMessagesWithXmlAttachment) throws MessagingException {

    listMessages.removeAll(listMessagesWithXmlAttachment);

    for (Message message : listMessages) {
        message.setFlag(Flags.Flag.DELETED, true);
    }

}

private static void moveMessagesToFolders(List<Message> listMessages, Store store) throws MessagingException {

    HashMap<String, List<Message>> mapMessages = separeteMessagesBySender(listMessages);
    Folder folder = null;

    for (Entry<String, List<Message>> mapMessage : mapMessages.entrySet()) {

        Message[] messageArray = mapMessage.getValue().toArray(new Message[mapMessage.getValue().size()]);
        folder = store.getDefaultFolder().getFolder(mapMessage.getKey());
        folder.open(Folder.READ_WRITE);
        folder.appendMessages(messageArray);
        EmailUtil.deleteListMessage(mapMessage.getValue());

    }
}

private static HashMap<String, List<Message>> separeteMessagesBySender(List<Message> listMessages) throws MessagingException {

    HashMap<String, List<Message>> mapMessages = new HashMap<>();
    List<Message> listMessageSeparetad = null;

    for (Message message : listMessages) {

        String sender = EmailUtil.getSender(message);

        if (!mapMessages.containsKey(sender)) {
            listMessageSeparetad = new ArrayList<>();
            mapMessages.put(sender, listMessageSeparetad);
        }

        listMessageSeparetad = mapMessages.get(sender);
        listMessageSeparetad.add(message);
    }

    return mapMessages;
}
\$\endgroup\$
0
5
\$\begingroup\$

I have a feeling your application is slow because it does a lot of network access ... ;-)

The amount of time in your code will be a very small fraction of the actual 'latency'. So, the question is not "How can we make your code faster?" but rather it is "How can we reduce the amount of network traffic?"

I am not very well versed with managing the performance of the javamail API. When I have used it the Mail servers have been local, and not really a factor in performance... But, without actually trying it myself, you should be using the fetch(...,...) method.

Also, there's no real need for the outside if (EmailUtil.hasMessages(inbox))....

Sometimes it is better to explain with code, rather than with blurb:

// clear out any delete-marked messages - no need to process them....
inbox.expunge();
// by convention, this should be a lightweight process...
// hopefully GMail does it right.
Message[] messages = inbox.getMessages();
if (messages.length > 0) {
    Message[] todelete = new Message[messages.length];
    deletecount = 0;
    // now, instead of doing a loop through all the messages,
    // do a bulk 'get' operation to get the fields we know we will need:
    FetchProfile profile = new FetchProfile();
    // add the headers we know we will need:
    profile.add(FetchProfile.Item.CONTENT_INFO);
    // this is a bulk operation that should get the content-type,
    // and other things.
    inbox.fetch(messages, profile);
    for (int m = 0; m < messages.length; m++) {
        if (EmailUtil.hasNoAttachment(messages[m].getContentType())) {
            todelete[deletecnt++] = messages[m];
            messages[m] = null;
        }
    }
    // OK, do a bulk delete operation
    if (deletecnt > 0) {
        todelete = Arrays.copyOf(todelete, deletecnt);
        Flags delflags = new Flags(Flags.Flag.DELETED);
        inbox.setFlags(todelete, delflags, true);
        inbox.expunge();
    }

}

From this you can get the idea of what you should do....

I realize I kept the Message[] as an array, and you have it as a List. I kept it as an array because it is the input mechanism for the bulk methods of the API... but, in fairness, I think you should keep using the List format (the iterator.remove() is useful). Then, convert the List to an array when you need it.

Also, look in to the FetchProfile object/method, you can add any header fields you want to it...

Also, you can filter out messages that do not meet certain criteria, and then fetch additional data for the next set of tests, and do it that way.

Bottom line is that you want to access the server as few times as possible, and when you access it, you should get as little data as you need, but, group all similar requests in to a sigle operation.....

\$\endgroup\$
1
  • \$\begingroup\$ Sorry, but I forgot to view this post, I´ve update my code today including some threads...Yes my problem is to reduce time access to mail box for my operation \$\endgroup\$ Dec 9 '13 at 0:10
1
\$\begingroup\$

After finally reading the advice of @rolfl and doing my own testing, I came up with this code. I hope someone can use or improve my code and share. I don't request any further reviews of it, though.

This code has some methods that I put in util´s classes but the name tells what he do.

public class Email {

    private static List<File> listFile = new ArrayList<>();
    private static final int threadQuantity = ThreadUtil.quantityOfThreads();

    public static void main(String[] args) {

        try {

            Properties properties = System.getProperties();
            properties.setProperty("mail.store.protocol", "imaps");
            properties.put("mail.imap.fetchsize", "819200");            
            properties.setProperty("mail.imap.partialfetch", "false");
            properties.setProperty("mail.imaps.partialfetch", "false");         
            Session session = Session.getDefaultInstance(properties, null);
            Store store = session.getStore("imaps");
            store.connect("imap.gmail.com", "login", "password");

            Folder inbox = store.getDefaultFolder();
            inbox = inbox.getFolder("INBOX");
            inbox.open(Folder.READ_WRITE);

            FetchProfile fetchProfile = new FetchProfile();
            fetchProfile.add(FetchProfile.Item.CONTENT_INFO);

            Message[] messages = inbox.getMessages();
            Message[] messagesToDelete = new Message[messages.length];
            int countToDelete = 0;

            inbox.fetch(messages, fetchProfile);

            countToDelete += EmailUtil.separteMessagesWithoutAttachment(messages, messagesToDelete);
            countToDelete += separteMessagesWithoutXmlAttachment(messages, messagesToDelete);

            if (hasMessagesWithoutXmlAttachment(countToDelete)) {
                deleteMessagesWithoutXmlAttachment(messagesToDelete, countToDelete, inbox);
                messages = separateMessagesWithAttachment(messages, countToDelete);
            }

            Set<String> setSender = getMessagesSender(messages);
            Set<String> setFolderName = getFoldersName(store);

            if (foldersNotExist(setFolderName, setSender)) {
                createFolder(store, setSender);
            }

            moveMessagesToFolders(messages, store, setSender);
            deleteMessages(messages, inbox);

        } catch (MessagingException | IOException e) {
            e.printStackTrace();
        }
    }

    private static int separteMessagesWithoutXmlAttachment(Message[] messages, Message[] messagesToDelete) throws IOException, MessagingException {

        List<MimeBodyPart> listMimeBodyPart = new ArrayList<>();
        int countToDelete = 0;

        for (int i = 0; i < messages.length; i++) {

            if (!NullUtil.isNull(messages[i])){
                if (hasNoXmlAttachment(messages[i], listMimeBodyPart)) {
                    messagesToDelete[countToDelete++] = messages[i];
                    messages[i] = null;
                }               
            }
        }
        processAttachmentXml(listMimeBodyPart);

        return countToDelete;
    }

    private static boolean hasNoXmlAttachment(Message message, List<MimeBodyPart> listMimeBodyPart) throws IOException, MessagingException {

        Multipart multipart = (Multipart) message.getContent();
        boolean hasNoXmlAttachment = true;

        for (int i = 0; i < multipart.getCount(); i++) {

            MimeBodyPart mimeBodyPart = (MimeBodyPart) multipart.getBodyPart(i);

            if (Part.ATTACHMENT.equalsIgnoreCase(mimeBodyPart.getDisposition())) {

                if (FileUtil.isXmlFile(mimeBodyPart.getFileName())) {
                    hasNoXmlAttachment = false;
                    listMimeBodyPart.add(mimeBodyPart);

                }
            }
        }

        return hasNoXmlAttachment;
    }

    private static void processAttachmentXml(List<MimeBodyPart> listMimeBodyPart) throws MessagingException, IOException {

        MimeBodyPart[] mimeBodyPartsArray = listMimeBodyPart.toArray(new MimeBodyPart[listMimeBodyPart.size()]);
        List<Thread> listThread = ThreadUtil.creatPoolThread();
        int[] indexThread = ThreadUtil.generateThreadsIndex(mimeBodyPartsArray);

        int indexStart = 0;
        int indexEnd = 0;

        for (int i = 0; i < threadQuantity; i++) {
            indexEnd += indexThread[i];
            MimeBodyPart[] mimeBodyPartsArrayCopy = (MimeBodyPart[]) ThreadUtil.sliceArray(mimeBodyPartsArray, indexStart, indexEnd);
            indexStart = indexEnd;
            listThread.add(new ThreadGenerateFile(mimeBodyPartsArrayCopy, listFile));
        }

        ThreadUtil.startThreads(listThread);
    }

    private static boolean hasMessagesWithoutXmlAttachment(int countToDelete) {
        return countToDelete > 0;
    }

    private static void deleteMessagesWithoutXmlAttachment(Message[] messagesToDelete, int countToDelete, Folder inbox) throws MessagingException {

        messagesToDelete = Arrays.copyOf(messagesToDelete, countToDelete);
        Flags flagToDelete = new Flags(Flag.DELETED);

        inbox.setFlags(messagesToDelete, flagToDelete, true);
        inbox.expunge();
    }

    private static Message[] separateMessagesWithAttachment(Message[] messages, int countToDelete) {

        List<Message> listMessage = new ArrayList<>(Arrays.asList(messages));
        listMessage.removeAll(Collections.singleton(null));
        return listMessage.toArray(new Message[listMessage.size()]);
    }

    private static Set<String> getMessagesSender(Message[] messages) throws MessagingException {

        Set<String> setSender = new HashSet<>();

        for (Message message : messages) {
            if(!NullUtil.isNull(message)){
                setSender.add(EmailUtil.getSender(message));
            }
        }

        return setSender;
    }

    private static Set<String> getFoldersName(Store store) throws MessagingException {

        Set<Folder> setFolder = new HashSet<>(Arrays.asList(store.getDefaultFolder().list()));
        Set<String> setFolderName = new HashSet<>();

        for (Folder folder : setFolder) {
            setFolderName.add(folder.getName());
        }

        return setFolderName;
    }

    private static void createFolder(Store store, Set<String> setSenders) throws MessagingException {

        String[] senderArray = setSenders.toArray(new String[setSenders.size()]);

        List<Thread> listThread = ThreadUtil.creatPoolThread();
        int[] indexThread = ThreadUtil.generateThreadsIndex(senderArray);

        int indexStart = 0;
        int indexEnd = 0;

        for (int i = 0; i < threadQuantity; i++) {
            indexEnd += indexThread[i];
            String[] senderArrayCopy = (String[]) ThreadUtil.sliceArray(senderArray, indexStart, indexEnd);
            indexStart = indexEnd;
            listThread.add(new ThreadCreateFolder(senderArrayCopy, store));
        }

        ThreadUtil.startThreads(listThread);
    }

    private static boolean foldersNotExist(Set<String> setFolderName, Set<String> setSenders) throws MessagingException {
        return !(setFolderName.containsAll(setSenders));
    }

    private static void moveMessagesToFolders(Message[] messages, Store store, Set<String> setSender) throws MessagingException {

        HashMap<String, List<Message>> mapMessages = separeteMessagesBySender(messages, setSender);

        for (Entry<String, List<Message>> mapMessage : mapMessages.entrySet()) {

            Message[] messageArray = mapMessage.getValue().toArray(new Message[mapMessage.getValue().size()]);

            List<Thread> listThread = ThreadUtil.creatPoolThread();
            int[] indexThread = ThreadUtil.generateThreadsIndex(messageArray);

            int indexStart = 0;
            int indexEnd = 0;

            for (int i = 0; i < threadQuantity; i++) {
                indexEnd += indexThread[i];
                Message[] messageArrayCopy = (Message[]) ThreadUtil.sliceArray(messageArray, indexStart, indexEnd);
                indexStart = indexEnd;
                listThread.add(new ThreadMoveMessages(messageArrayCopy, mapMessage.getKey(), store));
            }

            ThreadUtil.startThreads(listThread);
        }
    }

    private static HashMap<String, List<Message>> separeteMessagesBySender(Message[] messages, Set<String> setSender) throws MessagingException {

        HashMap<String, List<Message>> mapMessages = new HashMap<>();
        List<Message> listMessage = null;

        for (String sender : setSender) {
            listMessage = new ArrayList<>();
            mapMessages.put(sender, listMessage);
        }

        for (Message message : messages) {
            if (!NullUtil.isNull(message)) {
                listMessage = mapMessages.get(EmailUtil.getSender(message));
                listMessage.add(message);               
            }
        }

        return mapMessages;
    }

    private static void deleteMessages(Message[] messages, Folder inbox) throws MessagingException {
        inbox.setFlags(messages, new Flags(Flag.DELETED), true);
        inbox.expunge();
    }

}
\$\endgroup\$
5
  • \$\begingroup\$ Thanks for posting your revisions back here. Can you give some indication of how much your performance improved ... was it just a 'little' or was it a lot? \$\endgroup\$
    – rolfl
    Dec 15 '13 at 18:46
  • \$\begingroup\$ I can´t tell how much improved, but what you told me to do improved... \$\endgroup\$ Dec 16 '13 at 17:47
  • \$\begingroup\$ @rolfl I´ve tested my code with sequencial and parallel, my test was with around 400 e-mails with attatchments or no, and with threads I got less 4 minutos than sequencial. I got in sequencial 19 minutes and parallel 15 minutes. \$\endgroup\$ Dec 31 '13 at 7:40
  • \$\begingroup\$ I advise you to change your Google account password immediately! \$\endgroup\$ Dec 31 '13 at 8:06
  • 1
    \$\begingroup\$ Furthermore, this being Code Review, I'll point out that usernames and passwords should be stored in configuration files, not in the source code. \$\endgroup\$ Dec 31 '13 at 8:08

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.