2
\$\begingroup\$

Here's my updated version of the problem I posted at Calculating time windows for entities, adding in suggested changes as well as a change I did so that the Reflection is 'cached' in a ConcurrentDictionary and happens only once per Type.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Reflection;

namespace Services.Helpers
{
    #region Custom Attributes
    [AttributeUsage(AttributeTargets.Property)]
    public class DoNotCopyIntoTimeWindow : Attribute { } // leave default

    [AttributeUsage(AttributeTargets.Property)]
    public class IsProcessedIntoTimeWindow : Attribute { } // calculate time window for this property

    [AttributeUsage(AttributeTargets.Property)]
    public class IsTimeWindowDate : Attribute { } // attribute to mark property as the datetime

    [AttributeUsage(AttributeTargets.Property)]
    public class IsTimeWindowIdentifier : Attribute { } // this is the time window property
    #endregion

    public class TimeWindow
    {
        #region Structs
        public struct TimeWindowDictionary
        {
            public PropertyInfo PropertyInfo { get; set; }
            public Dictionary<NullObject<dynamic>, int> Dictionary { get; set; }
        }

        public struct NullObject<T>
        {
            [DefaultValue(true)]
            private readonly bool isnull; // default property initializers are not supported for structs

            private NullObject(T item, bool isnull) : this()
            {
                this.isnull = isnull;
                Item = item;
            }

            public NullObject(T item) : this(item, item == null)
            {
            }

            public static NullObject<T> Null()
            {
                return new NullObject<T>();
            }

            public T Item { get; private set; }

            public bool IsNull()
            {
                return isnull;
            }

            public static implicit operator T(NullObject<T> nullObject)
            {
                return nullObject.Item;
            }

            public static implicit operator NullObject<T>(T item)
            {
                return new NullObject<T>(item);
            }

            public override string ToString()
            {
                return (Item != null) ? Item.ToString() : "NULL";
            }

            public override bool Equals(object obj)
            {
                if (obj == null)
                    return IsNull();

                if (!(obj is NullObject<T>))
                    return false;

                var no = (NullObject<T>)obj;

                if (IsNull())
                    return no.IsNull();

                if (no.IsNull())
                    return false;

                return Item.Equals(no.Item);
            }

            public override int GetHashCode()
            {
                if (IsNull())
                    return 0;

                var result = Item.GetHashCode();

                if (result >= 0)
                    result++;

                return result;
            }
        }

        public struct Properties
        {
            public List<PropertyInfo> PropertiesToProcess { get; set; }
            public List<PropertyInfo> CopyProperties { get; set; }
            public PropertyInfo TimeWindowIdentifier { get; set; }
            public PropertyInfo DatePropertyInfo { get; set; }
            public int Size { get; set; }
        }
        #endregion

        #region Class Members
        private static readonly ConcurrentDictionary<Type, Properties> PropertiesDictionary = new ConcurrentDictionary<Type, Properties>();
        #endregion

        #region Methods
        public static IEnumerable<T> CalculateTimeWindows<T>(DateTime dateFrom, DateTime dateTo, List<T> stateModels) where T : new()
        {
            if (stateModels.Count() == 0)
                return new List<T>();

            dateFrom = GetPropertiesAndDictionaries(
              dateFrom,
              stateModels,
              out PropertyInfo datePropertyInfo,
              out List<PropertyInfo> copyProperties,
              out PropertyInfo timeWindowIdentifier,
              out int size,
              out TimeWindowDictionary[] dictionaries);

            byte[] windowDurations = { 5, 15, 60 };

            return windowDurations.SelectMany(wd =>
              CalculateTimeWindow(
                dateFrom,
                dateTo,
                stateModels,
                wd,
                datePropertyInfo,
                copyProperties,
                timeWindowIdentifier,
                size,
                dictionaries));
        }

        public static IEnumerable<T> CalculateTimeWindow<T>(DateTime dateFrom, DateTime dateTo, List<T> stateModels, byte timeWindowMinutes, PropertyInfo datePropertyInfo, List<PropertyInfo> copyProperties, PropertyInfo timeWindowIdentifier, int size, TimeWindowDictionary[] dictionaries) where T : new()
        {
            if (stateModels.Count() > 0)
            {
                DateTime currentWindowFrom, currentWindowTo, nextWindowFrom;
                nextWindowFrom = dateFrom;
                int itemPointer = 0;
                T prevItem = default;
                T prevTimeWindow = default;

                int j = 1;

                do // one time window
                {
                    for (int i = 0; i < size; i++)
                        dictionaries[i].Dictionary = new Dictionary<NullObject<dynamic>, int>();

                    currentWindowFrom = nextWindowFrom;
                    nextWindowFrom = currentWindowFrom.AddMinutes(timeWindowMinutes);
                    currentWindowTo = nextWindowFrom.AddSeconds(-1);

                    var calculateTime = currentWindowFrom;

                    for (; itemPointer < stateModels.Count(); itemPointer++)
                    {
                        var item = stateModels.ElementAt(itemPointer);

                        var date = (DateTime)datePropertyInfo.GetValue(item);

                        if (date >= currentWindowTo)
                            break;

                        var endDate = (date > currentWindowTo) ? nextWindowFrom : date; // state might extend more than the end of the time window
                        CalculateStateSeconds(prevItem, dictionaries, calculateTime, endDate);

                        prevItem = item;
                        calculateTime = (date < currentWindowFrom) ? currentWindowFrom : date; // to fix the 'yesterday' date
                    }

                    if (calculateTime < currentWindowTo)
                        CalculateStateSeconds(prevItem, dictionaries, calculateTime, nextWindowFrom);

                    if (dictionaries[0].Dictionary.Count > 0)
                    {
                        bool sameAsPrevious = (prevTimeWindow != null);
                        var output = new T();

                        foreach (var dictionary in dictionaries)
                        {
                            var maxValue = dictionary.Dictionary.First();
                            for (j = 1; j < dictionary.Dictionary.Count; j++)
                            {
                                var valuePair = dictionary.Dictionary.ElementAt(j);

                                if (valuePair.Value > maxValue.Value)
                                    maxValue = valuePair;
                            }
                            var valToSet = maxValue.Key.Item;

                            if (sameAsPrevious)
                            {
                                var prevVal = GetValue(prevTimeWindow, dictionary.PropertyInfo);

                                if (!(valToSet == null && prevVal == null))
                                    sameAsPrevious = (valToSet == prevVal);
                            }

                            SetValue(output, dictionary.PropertyInfo, valToSet);
                        }

                        if (!sameAsPrevious)
                        {
                            foreach (var copyProperty in copyProperties)
                                SetValue(output, copyProperty, copyProperty.GetValue(prevItem));

                            timeWindowIdentifier.SetValue(output, timeWindowMinutes);

                            datePropertyInfo.SetValue(output, currentWindowFrom);

                            prevTimeWindow = output;

                            yield return output;
                        }
                    }
                }
                while (nextWindowFrom <= dateTo);
            }
        }

        private static DateTime GetPropertiesAndDictionaries<T>(DateTime dateFrom, List<T> stateModels, out PropertyInfo datePropertyInfo, out List<PropertyInfo> copyProperties, out PropertyInfo timeWindowIdentifier, out int size, out TimeWindowDictionary[] dictionaries) where T : new()
        {
            Type tType = typeof(T);
            if (!PropertiesDictionary.TryGetValue(tType, out Properties properties))
            {
                var propInfos = tType.GetProperties();

                datePropertyInfo = propInfos.Single(p => p.GetCustomAttributes(typeof(IsTimeWindowDate), true).Any());

                var propertiesToProcess = propInfos.Where(p => p.GetCustomAttributes(typeof(IsProcessedIntoTimeWindow), true).Any()).ToList();

                copyProperties = propInfos.Where(p => !p.GetCustomAttributes(typeof(IsTimeWindowIdentifier), true).Any() && !p.GetCustomAttributes(typeof(DoNotCopyIntoTimeWindow), true).Any() && !p.GetCustomAttributes(typeof(IsTimeWindowDate), true).Any() && !p.GetCustomAttributes(typeof(IsProcessedIntoTimeWindow), true).Any() && p.CanWrite && !p.GetMethod.IsVirtual).ToList();
                timeWindowIdentifier = propInfos.Single(p => p.GetCustomAttributes(typeof(IsTimeWindowIdentifier), true).Any());

                size = propertiesToProcess.Count();
                properties = new Properties()
                {
                    CopyProperties = copyProperties,
                    DatePropertyInfo = datePropertyInfo,
                    PropertiesToProcess = propertiesToProcess,
                    TimeWindowIdentifier = timeWindowIdentifier,
                    Size = size
                };
                PropertiesDictionary.TryAdd(tType, properties);
            }
            else
            {
                datePropertyInfo = properties.DatePropertyInfo;
                copyProperties = properties.CopyProperties;
                timeWindowIdentifier = properties.TimeWindowIdentifier;
                size = properties.Size;
            }

            dictionaries = properties.PropertiesToProcess
                                    .Select(p => new TimeWindowDictionary { PropertyInfo = p })
                                    .ToArray();

            var firstDate = (DateTime)datePropertyInfo.GetValue(stateModels.First());

            if (firstDate < dateFrom)
                dateFrom = new DateTime(firstDate.Year, firstDate.Month, firstDate.Day, firstDate.Hour, 0, 0, DateTimeKind.Utc);

            return dateFrom;
        }

        private static dynamic GetValue(object inputObject, PropertyInfo propertyInfo)
        {
            return propertyInfo.GetValue(inputObject);
        }

        //private static void SetValue(object inputObject, string propertyName, object propertyVal)
        private static void SetValue(object inputObject, PropertyInfo propertyInfo, object propertyVal)
        {
            if (propertyVal != null)
            {
                //find the property type
                Type propertyType = propertyInfo.PropertyType;

                //Convert.ChangeType does not handle conversion to nullable types
                //if the property type is nullable, we need to get the underlying type of the property
                var targetType = IsNullableType(propertyType) ? Nullable.GetUnderlyingType(propertyType) : propertyType;

                //Returns an System.Object with the specified System.Type and whose value is
                //equivalent to the specified object.
                propertyVal = Convert.ChangeType(propertyVal, targetType);
            }

            //Set the value of the property
            propertyInfo.SetValue(inputObject, propertyVal, null);

        }

        private static bool IsNullableType(Type type)
        {
            return type.IsGenericType && type.GetGenericTypeDefinition().Equals(typeof(Nullable<>));
        }

        private static void CalculateStateSeconds<T>(T prevItem, IEnumerable<TimeWindowDictionary> dictionaries, DateTime calculateTime, DateTime endDate)
        {
            if (prevItem != null)
            {
                var seconds = Convert.ToInt32(endDate.Subtract(calculateTime).TotalSeconds);

                foreach (var dictionary in dictionaries)
                {
                    var key = dictionary.PropertyInfo.GetValue(prevItem);
                    dictionary.Dictionary.TryGetValue(key, out int existingSeconds);
                    dictionary.Dictionary[key] = existingSeconds + seconds;
                }
            }
        }
        #endregion
    }
}

Now the problem I'm having is that when I call the method in this way:

var stopWatchTW = new Stopwatch();
stopWatchTW.Start();
CalculateTimeWindows();
stopWatchTW.Stop();
ConsoleLogger.WriteLine($"Processing time windows took {stopWatchTW.ElapsedMilliseconds}ms");

private void CalculateTimeWindows()
{
     myList1.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList1).ToList());
     myList2.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList2).ToList());
     myList3.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList3).ToList());
     myList4.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList4).ToList());
     myList5.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList5).ToList());
     myList6.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList6).ToList());
     myList7.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList7).ToList());
     myList8.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList8).ToList());
     myList9.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList9).ToList());
     myList10.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList10).ToList());
     myList11.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList11).ToList());
     myList12.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList12).ToList());
     myList13.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList13).ToList());
     myList14.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList14).ToList());
     myList15.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList15).ToList());
     myList16.AddRange(TimeWindow.CalculateTimeWindows(dateFrom, dateTo, myList16).ToList());
}

where each myList would be an empty list, the code is taking a very long time to run when deployed as an Azure WebJob (4.5 to 8 SECONDS to run), despite that I have this at the very top of the CalculateTimeWindows method:

if (stateModels.Count() == 0)
    return new List<T>();
\$\endgroup\$
  • \$\begingroup\$ It can't possible be a problem with CalculateTimeWindows. What if you only have one or two lists, does that reduce the latency?Do you have any time consuming static initialization on the data model or in some attributes? \$\endgroup\$ – Henrik Hansen Jun 18 '19 at 8:58
  • \$\begingroup\$ When I debug the code on my machine, I never experience such delays. It's only when it's in the production environment deployed as an Azure WebJob that this happens. I can't test with 2 lists, it needs to be all 16 because it's exactly in a production environment. I'm not sure what you mean by the static initialization? To note that this process (of doing these 16 calls) doesn't happen just once, but I'm doing it repeatedly, thousands of times. It's not the first one which is slow; it remains slow. \$\endgroup\$ – Mark Cilia Vincenti Jun 18 '19 at 9:02
  • 1
    \$\begingroup\$ You don't really have 16 lines of the same code, do you? \$\endgroup\$ – t3chb0t Jun 18 '19 at 9:53
  • 2
    \$\begingroup\$ How do you know it's exactly this method that is slow? Have you measured it? It is virtually impossible that it takes that much time with that empty-list guard at the top. \$\endgroup\$ – t3chb0t Jun 18 '19 at 10:49
  • 1
    \$\begingroup\$ Calling it nullable-key would be much wiser and more intuitive. \$\endgroup\$ – t3chb0t Jun 18 '19 at 13:14
1
\$\begingroup\$

Well, this is embarrassing. Turns out, that the lists weren't empty after all, but they had 1 item, with its date property (having the attribute IsTimeWindowDate) preceding the dateFrom. The result of CalculateTimeWindows would still be an empty list, and at the end I was only counting the rows inserted in the database (taking for example myList1.Where(x => x.Date >= dateFrom)).

Still, this is taking a wee bit too much for 1 row.

EDIT: An improvement with this code whilst debugging locally, but actually taking longer when deployed live on Azure:

(first setting lastWindow = false; inside the CalculateTimeWindows)

                    ...
                    if (itemPointer == stateModels.Count())
                        lastWindow = true;
                    else if (lastWindow)
                        break;
                }
                while (nextWindowFrom <= dateTo);
| improve this answer | |
\$\endgroup\$
  • 4
    \$\begingroup\$ lol, this is why you always should post all relevant code, maybe next time ;-) \$\endgroup\$ – t3chb0t Jun 18 '19 at 13:14
  • \$\begingroup\$ aye @t3chb0t, still it doesn't explain why 16 lists with 1 item each, all preceding the dateFrom, should take up to 8 seconds though, does it? \$\endgroup\$ – Mark Cilia Vincenti Jun 18 '19 at 13:20
  • \$\begingroup\$ In fact, my updated Unit test is now taking 44ms during debugging. Never anywhere close to 8 seconds. \$\endgroup\$ – Mark Cilia Vincenti Jun 18 '19 at 13:32
  • \$\begingroup\$ Just now, even with the improvement in the edit of this answer, the 16 calls to CalculateTimeWindows which yielded 0 rows took 12344ms; that's over 12 seconds. \$\endgroup\$ – Mark Cilia Vincenti Jun 18 '19 at 14:54
  • \$\begingroup\$ Run it multiple times within the same session. First unmeasured then with the benchmark. There might be some initialization or JIT time added to the result. \$\endgroup\$ – t3chb0t Jun 18 '19 at 14:57

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.