2
\$\begingroup\$

I have a list which has 25000 data and I am iterating in all data and in each iteration, I am searching data in another data table. my whole routine is taking a long time to finish.

Loading data from XML file into a dataset.

If the ignoredata field is there in QCCommentstmp dataset table QCCommentstmp.Tables[0].Columns.Contains("IgnoreData") then deserialize QCCommentstmp table data to list List<clsCommentPopup> QCCommentlist.

Iterate in QCCommentlist's data using a for loop and finding data in QCViewAlltmp.Tables[0] data table for each iteration.

When QCCommentlist has 25000 data then I am iterating in all 25000 data and finding data in another data table. If data is found then I am updating data in the list. This process is getting very slow and the code is taking a long time to finish all the iteration and searching data in the data table.

Please review my code and tell me how to restructure my code, as a result, there will be the improvement in code execution speed.

If my approach is wrong then guide me with the right approach and also give me the relevant code which I can use in my above code, as a result, my routine will take minimum time to finish if I iterate in more than 25000 data. I'm looking for suggestions and better code to achieve the same task.

Code

private void UpdateCommentFirst(string strCommentPath, string TickerName)
{
    int counter = 0;
    bool QcViewAllFileExist = false;
    bool QcCommentFileExist = false;
    bool AllowUpdate = false;
    string savepath = Convert.ToString(ConfigurationManager.AppSettings["OutputPath"]).Trim() + TickerName + "\\" + TickerName + "_QC-ViewwAll.xml";
    DataSet QCCommentstmp = new DataSet();
    DataSet QCViewAlltmp = new DataSet();

    if (File.Exists(strCommentPath))
    {
        QCCommentstmp.ReadXml(strCommentPath);
        QcCommentFileExist = true;
    }

    if (File.Exists(savepath))
    {
        QCViewAlltmp.ReadXml(savepath);
        QcViewAllFileExist = true;
    }

    if (QcCommentFileExist && QcViewAllFileExist)
    {
        if (QCCommentstmp.Tables.Count > 0)
        {
            if (!QCCommentstmp.Tables[0].Columns.Contains("IgnoreData"))
            {
                AllowUpdate = true;
            }
        }

        if (AllowUpdate)
        {

            List<clsCommentPopup> QCCommentlist = QCCommentstmp.Tables[0].AsEnumerable()
                .Select(row => new clsCommentPopup
                {
                    //BrokerFor,Formula,LineItem,Section,PeriodCollection
                    bolFollowUP = (row.Field<string>("FollowUP")) == null ? false : Convert.ToBoolean((row.Field<string>("FollowUP"))),
                    bolThisPeriod = (row.Field<string>("ThisPeriod")) == null ? false : Convert.ToBoolean((row.Field<string>("ThisPeriod"))),
                    Formula = (row.Field<string>("Formula")) == null ? string.Empty : (row.Field<string>("Formula")),
                    ModelValue = (row.Field<string>("ModelValue")) == null ? string.Empty : (row.Field<string>("ModelValue")),
                    ExternalComment = (row.Field<string>("ExternalComment")) == null ? string.Empty : (row.Field<string>("ExternalComment")),
                    InternalComment = (row.Field<string>("InternalComment")) == null ? string.Empty : (row.Field<string>("InternalComment")),
                    strEndPeriod = (row.Field<string>("EndPeriod")) == null ? string.Empty : (row.Field<string>("EndPeriod")),
                    strStartPeriod = (row.Field<string>("StartPeriod")) == null ? string.Empty : (row.Field<string>("StartPeriod")),
                    PeriodType = (row.Field<string>("PeriodType")) == null ? string.Empty : (row.Field<string>("PeriodType")),
                    SectionFor = (row.Field<string>("Section")) == null ? string.Empty : (row.Field<string>("Section")),
                    LiFor = (row.Field<string>("LineItem")) == null ? string.Empty : (row.Field<string>("LineItem")),
                    QcPeriodFor = (row.Field<string>("QcPeriod")) == null ? string.Empty : (row.Field<string>("QcPeriod")),
                    BrokerFor = (row.Field<string>("BrokerFor")) == null ? string.Empty : (row.Field<string>("BrokerFor")),
                    PeriodCollection = (row.Field<string>("PeriodCollection")) == null ? string.Empty : (row.Field<string>("PeriodCollection")),
                    boolIgnoreValue = (row.Field<string>("IgnoreValue")) == null ? false : Convert.ToBoolean((row.Field<string>("IgnoreValue"))),
                    IgnoreData = (!QCCommentstmp.Tables[0].Columns.Contains("IgnoreData") ? string.Empty : (row.Field<string>("IgnoreData") == null ? string.Empty : row.Field<string>("IgnoreData")))
                }).ToList();


            if (QCCommentlist != null)
            {
                foreach (var comment in QCCommentlist)
                {
                    string section = comment.SectionFor;
                    string li = comment.LiFor;
                    string broker = comment.BrokerFor;
                    string period = comment.PeriodCollection;
                    string strQCPeriodValue = "";

                    if (comment.boolIgnoreValue && period.Trim() != "")
                    {
                        var QcViewColumnName = QCViewAlltmp.Tables[0].Columns.Cast<DataColumn>().AsParallel()
                          .Where(x => x.ColumnName.Contains(period))
                          .Select(x => new { x.ColumnName }).FirstOrDefault();

                        if (QcViewColumnName != null)
                        {
                            period = QcViewColumnName.ColumnName;

                            if (period.Trim() != "")
                            {
                                var datarow = QCViewAlltmp.Tables[0].AsEnumerable().AsParallel()
                                    .Where(row => row.Field<string>("GroupKey").Split('~')[0].ToUpper() == section.ToUpper()
                                    && row.Field<string>("GroupKey").Split('~')[1].ToUpper() == li.ToUpper()
                                    && row.Field<string>("Section ").ToUpper() == broker.ToUpper());

                                if (datarow != null && datarow.Count() > 0)
                                {
                                    strQCPeriodValue = (datarow.FirstOrDefault()[period] != null ? datarow.FirstOrDefault()[period].ToString() : string.Empty);
                                    if (strQCPeriodValue.Trim() != string.Empty)
                                    {
                                        comment.IgnoreData = strQCPeriodValue;
                                        counter++;
                                    }
                                }
                            }
                        }
                    }
                }
            }

            SerializeQcComment(QCCommentlist);
            toolTip1.Hide(this);
        }
    }
}
\$\endgroup\$
  • 2
    \$\begingroup\$ Break this up into smaller functions and find where you are spending the most time. This function is too complex. \$\endgroup\$ – pacmaninbw Apr 7 at 0:26
  • \$\begingroup\$ should i use parallel foreach instead of normal foreach to get better performance? \$\endgroup\$ – user197025 Apr 7 at 18:16
  • 1
    \$\begingroup\$ The only way you'll know which is faster is if you time it: we don't have the data, so we can't really answer that question. We might be able to suggest how it might be faster, but in the end you have to do the measurements and make a call. \$\endgroup\$ – VisualMelon Oct 8 at 11:56

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.