2
\$\begingroup\$

I have the follow script to change my data within Mongo DB:

db.TRACKING_DATA.find( { 'eventdata' : { $type : 2 } } ).forEach( 
function (x) { x.eventdata = JSON.parse(x.eventdata); 
    db.TRACKING_DATA.save(x);});

db.TRACKING_DATA.find().forEach(
    function(doc) {
    var QueryDate = doc.eventdata.QueryDate;
        if(QueryDate!=null)
        db.TRACKING_DATA.update({ 
            "_id": doc._id 
                }, { 
                    "$set": {
                            "eventdata.QueryDate": new Date(doc.eventdata.QueryDate)        
                        }
                }    
    ) 
});

db.TRACKING_DATA.find().forEach(
    function(doc) {    
        var Viewed = doc.eventdata.Viewed;
        if (Viewed instanceof Array && Viewed[0]) 
            db.TRACKING_DATA.update({ 
            "_id": doc._id ,"eventdata.Viewed.viewDate":doc.eventdata.Viewed[0].viewDate 
            },{ 
                "$set": {              
                        "eventdata.Viewed.$.viewDate": new Date( doc.eventdata.Viewed[0].viewDate)        
                        }
                }  
            ) 
        });

db.TRACKING_DATA.find().forEach(
    function(doc) {
        var Viewed = doc.eventdata.Viewed;
        if (Viewed instanceof Array && Viewed[0]) {
            var Message = Viewed[0].Message;
            if (Message instanceof Array && Message[0]) {

                db.TRACKING_DATA.update({
                    "_id" : doc._id,
                    "eventdata.Viewed.viewDate" : Viewed[0].viewDate
                }, {
                    "$set" : {
                        "eventdata.Viewed.$.Message.0.MessageDate" : new Date(Message[0].MessageDate)
                    }
                });
            }
        }
    });

 db.TRACKING_DATA.find().forEach(     
    function(doc) {         
        var Viewed = doc.eventdata.Viewed;         
            if (Viewed instanceof Array && Viewed[0]) {             
                var Message = Viewed[0].Message;             
                    if (Message instanceof Array && Message[0]) {                  
                        db.TRACKING_DATA.update({                     
                            "_id" : doc._id,                     
                                "eventdata.Viewed.viewDate" : Viewed[0].viewDate                 
                                }, {                     
                                "$set" : {                         
                                    "eventdata.Viewed.$.Message.0.Response.0.ResponseDate" : new Date(Message[0].Response[0].ResponseDate)                     
                                    }                 
                                });             
                            }         
                        }     
                    });

It runs, however, it is taking a very long time ~4hrs to change ~8million documents. How can I speed up the efficiency of this, to reduce the time it takes to run?

Here is the MongoDB structure.

{
_id: "fe50fdee-4ea3-4824-94af-f369633c0c7a",
_class: "com.tracking.daoservice.model.TrackingData",
modified: ISODate("2014-09-10T23:38:48.407Z"),
eventtype: "William-Test",
eventdata: {
    QueryDate: "01-APR-2014",
    SearchQuery: {
        keyword: "Java",
        location: "Santa Clara, CA",
        Facet: "skill~java~perl|workAuth~USC",
        SearchAgentId: "4299"
    },
    Viewed: [
        {
            ViewedID: "8992ade400a",
            Dockey: "3323aba3233",
            PID: "32399a",
            actionsTaken: "email|direct message|report seeker",
            viewDate: "01-APR-2014",
            MessageSent: "true",
            Message: [
                {
                    MessageID: "123aca323",
                    Delivered: "True",
                    Opened: "True",
                    ClickThroughRate: "NotBad",
                    MessageDate: "02-APR-2014",
                    Response: [
                        {
                            ResponseId: "a323a9da",
                            ResponseDate: "23-APR-2014"
                        }
                    ]
                }
            ]
        }
    ]
},
eventsource: "API-Dev Test - JMachine",
sourceip: "myIp",
entityid: "TmoneyBunnyWunny",
groupid: "Dice",
datecreated: ISODate("2014-09-10T23:38:48.405Z")
}
\$\endgroup\$
  • \$\begingroup\$ Welcome to Code Review! To make life easier for reviewers, please add sufficient context to your question. The more you tell us about what your code does and what the purpose of doing that is, the easier it will be for reviewers to help you. See also this meta question. In this case, we don't really have a clue about what data your database stores. \$\endgroup\$ – Simon Forsberg Oct 2 '14 at 22:24
1
\$\begingroup\$

Interesting question,

this should go much faster if you

  1. Have 1 find().forEach( only
  2. Do your 4 if statements inside that loop and update an object with the required values
  3. Do one single update per found record at the end with the object you were updating

Something this like ought to work ( again, assuming that criteria only requires _id )

db.TRACKING_DATA.find( { 'eventdata' : { $type : 2 } } ).forEach( 
function (x) { x.eventdata = JSON.parse(x.eventdata); 
    db.TRACKING_DATA.save(x);});

db.TRACKING_DATA.find().forEach( function(doc) {

  var searchCriteria = { "_id": doc._id  },
      set = {};

  //Check 1
  var QueryDate = doc.eventdata.QueryDate; 
  if( (QueryDate!=null) ){
    set["eventdata.QueryDate"] = new Date(doc.eventdata.QueryDate);
  }  

  //Check 2
  var Viewed = doc.eventdata.Viewed;
  if (Viewed instanceof Array && Viewed[0]){
    set["eventdata.Viewed.$.viewDate"] = new Date( doc.eventdata.Viewed[0].viewDate);       


    //Check 3
    var Message = Viewed[0].Message; 
    if (Message instanceof Array && Message[0]){     
      set["eventdata.Viewed.$.Message.0.MessageDate"] = new Date(Message[0].MessageDate);

      //Check 4
      var Response = Message[0].Response;       

      if( Response instanceof Array & Response[0]){
        set["eventdata.Viewed.$.Message.0.Response.0.ResponseDate"] = new Date(Message[0].Response[0].ResponseDate);
      }
    }
  }
  //Do we have anything to set ?
  if( Object.keys(set).length ){
    db.TRACKING_DATA.update( searchCriteria, { "$set" : set } );
  }
}); 

Note that this refactoring exercise brought out a possible flaw in your code. In check 4 in your code you are not checking for the Reponse object, you simply copy pasted the code from the 3rd check which only checks up to Message.

\$\endgroup\$
  • \$\begingroup\$ Hi have added a data JSON to give some insight into the structure of the data I am working on \$\endgroup\$ – Will Oct 2 '14 at 22:43
  • \$\begingroup\$ In response to item 3. I am not sure that it is possible given that inner doc of event data is free flowing meaning that it will not always contain all the inner values such as QueryData etc, which is why I split these up. \$\endgroup\$ – Will Oct 3 '14 at 1:43
  • \$\begingroup\$ Still looking at this, why do your search queries have criteria other than _id if _id is supposed to be unique? stackoverflow.com/questions/9932631/… \$\endgroup\$ – konijn Oct 3 '14 at 13:29

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.