A service to monitor CPU usage

Question

I have written a windows service that I am currently not that happy with as I am facing design issues (first windows service).

The aim of this service is to monitor CPU usage and send an alert when it reaches a threshold (coming from the DB).

So the user has set a threshold of > 60% CPU usage for 10 Seconds. Then the code below will need to check the CPU usage every second and if it's > 60% for 10 seconds send out an alert. Then send an alert once it's < 60%.

This is my current implementation:

Main Program

private static void Main(string[] args)
{
    HostFactory.Run(
        x =>
        {
            x.StartAutomatically();
            x.Service<HardwareMonitor>(
                s =>
                {
                    s.ConstructUsing(monitor => new HardwareMonitor());
                    s.WhenStarted(tc => tc.Start());
                    s.WhenStopped(tc => tc.Stop());
                });
            x.RunAsLocalSystem();
            x.EnableServiceRecovery(
                r =>
                {
                    r.RestartService(1);
                });

            x.SetDescription("NZBDash Monitor");
            x.SetDisplayName("NZBDash Monitor");
            x.SetServiceName("NZBDashMonitor");
            x.UseNLog();
    });
}

As you can see I am using Topshelf to schedule the service.

Hardware Monitor

public class HardwareMonitor
{
    private static void TaskManagerUnobservedTaskException(TaskExceptionInformation sender, UnhandledExceptionEventArgs e)
    {
        Console.WriteLine("An error happened with a scheduled task: " + e.ExceptionObject);
    }

    public void Start()
    {

        TaskManager.UnobservedTaskException += TaskManagerUnobservedTaskException;
        TaskManager.TaskFactory = new NinjectTaskFactory(ServiceKernel.GetKernel());
        TaskManager.Initialize(new TaskRegistry());
    }

    public void Stop()
    {
        TaskManager.Stop();
    }
}

The Hardware Monitor is using FluentScheduler to schedule different tasks

Task Registry

public class TaskRegistry : Registry
{
    public TaskRegistry()
    {
        Schedule<CpuMonitor>().ToRunNow();
    }
}

The main concern is in the CPU Monitor:

CPU Monitor

public class CpuMonitor : ITask, IRegisteredObject
{
    private readonly object _lock = new object();
    public int ThresholdPercentage { get; set; }
    public int TimeThresholdSec { get; set; }
    public int ThresholdBreachCount { get; set; }
    public DateTime BreachStart { get; set; }
    public DateTime BreachEnd { get; set; }
    private bool ShuttingDown { get; set; }
    private bool MonitoringEnabled { get; set; }
    private ISettingsService<HardwareSettingsDto> SettingsService { get; set; }
    private HardwareSettingsDto Settings { get; set; }
    private IEventService EventService { get; set; }
    private EmailAlert EmailAlert { get; set; }
    private ILogger Logger { get; set; }
    private ISmtpClient SmtpClient { get; set; }

    public CpuMonitor(ISettingsService<HardwareSettingsDto> settingsService, IEventService eventService, ILogger logger, ISmtpClient client)
    {
        Logger = logger;
        SmtpClient = client;
        EventService = eventService;
        SettingsService = settingsService;

        HostingEnvironment.RegisterObject(this);
        GetThresholds();

        if (!MonitoringEnabled)
            ShuttingDown = true;
    }

    private void GetThresholds()
    {
        Settings = SettingsService.GetSettings();
        MonitoringEnabled = Settings.EmailAlertSettings.AlertOnBreach || Settings.EmailAlertSettings.AlertOnBreachEnd;
        ThresholdPercentage = Settings.CpuMonitoring.CpuPercentageLimit;
        TimeThresholdSec = Settings.CpuMonitoring.ThresholdTime;
    }

    public void Alert()
    {
        EmailAlert = new EmailAlert(EventService, Logger,SmtpClient, Settings.EmailAlertSettings, BreachStart, BreachEnd);
        EmailAlert.Alert();
    }

    public void StartMonitoring()
    {
        Logger.Info("Starting CPU Monitor");
        try
        {
            using (var process = new PerformanceCounter("Processor", "% Processor Time", "_Total"))
            {
                var hasBeenBreached = false;
                while (true)
                {
                    hasBeenBreached = Monitor(process, hasBeenBreached);
                }
            }
        }
        catch (Exception e)
        {
            Logger.Fatal(e);
            Stop();

            //TODO: We need to possibly restart the service.
        }
    }

    public bool Monitor(PerformanceCounter process, bool hasBeenBreached)
    {
        Console.WriteLine("Monitoring");
        var breached = CheckBreach();
        if (breached)
        {
            hasBeenBreached = true;
            BreachStart = DateTime.Now;
            Alert();
        }
        else if (hasBeenBreached)
        {
            BreachEnd = DateTime.Now;
            EmailAlert = new EmailAlert(EventService, Logger,SmtpClient, Settings.EmailAlertSettings, BreachStart, BreachEnd);
            EmailAlert.Alert();
        }

        process.NextValue();
        Thread.Sleep(1000);
        var currentValue = process.NextValue();
        if (currentValue >= ThresholdPercentage)
        {
            ThresholdBreachCount++;
        }
        else
        {
            ThresholdBreachCount = 0;
        }

        return hasBeenBreached;
    }

    private bool CheckBreach()
    {
        // Check if the breach count >= than the time.
        // Each breach count should equal to 1 second
        return ThresholdBreachCount >= TimeThresholdSec;
    }

    public void Stop(bool immediate = false)
    {
        lock (_lock)
        {
            ShuttingDown = true;
        }
        Logger.Info("Stopping CPU Monitor");
        HostingEnvironment.UnregisterObject(this);
    }

    public void Execute()
    {
        lock (_lock)
        {
            if (ShuttingDown)
            {
                return;
            }

            StartMonitoring();
        }
    }
}

In the constructor we are going to the DB via the SettingsService (Injected via IoC Container) and then setting the thresholds. These thresholds are then never updated.

If the thresholds ever change (via Web UI) the only way to refresh this service is to restart it. I have thought about setting a timer and calling SettingsService.GetSettings() after a set amount of time but this seems like a smelly workaround.

Answer 1

OK, see below, it fetches configuration updates from the db. It is going to be a lot of code :)

First of all, we need a helper infrastructure base class to help compare configurations:

abstract class ValueObject<T> : IEquatable<ValueObject<T>>
    where T : ValueObject<T>
{
    protected abstract IEnumerable<object> EqualityCheckAttributes { get; }

    public override int GetHashCode() =>
        EqualityCheckAttributes
            .Aggregate(0, (hash, a) => hash = hash * 31 + (a?.GetHashCode() ?? 0));

    public override bool Equals(object obj) => 
        Equals(obj as ValueObject<T>);

    public bool Equals(ValueObject<T> other) =>
        other != null &&
            EqualityCheckAttributes.SequenceEqual(other.EqualityCheckAttributes);

    public static bool operator==(ValueObject<T> left, ValueObject<T> right) =>
        Equals(left, right);

    public static bool operator !=(ValueObject<T> left, ValueObject<T> right) =>
        !Equals(left, right);        
}

Infrastructure interfaces:

interface INotifier
{
    void Notify(bool critical); 
}

interface IPerformanceCounter
{
    double Value { get; } 
}

interface IConfigurationReader
{
    Configuration Read();
}

Where Configuration is:

class Configuration : ValueObject<Configuration>
{
    public Configuration(Intervals intervals, Thresholds thresholds)
    {
        Intervals = intervals;
        Thresholds = thresholds;
    }

    public Intervals Intervals { get; set; }
    public Thresholds Thresholds { get; }

    protected override IEnumerable<object> EqualityCheckAttributes => 
        new object[] { Intervals, Thresholds };
}

with Intervals:

class Intervals : ValueObject<Intervals>
{
    public Intervals(TimeSpan configuration, TimeSpan measurement, TimeSpan notification, TimeSpan criticalNotification)
    {
        Configuration = configuration;
        Measurement = measurement;
        Notification = notification;
        CriticalNotification = criticalNotification;
    }

    public TimeSpan Configuration { get; }
    public TimeSpan Measurement { get; }
    public TimeSpan Notification { get; }
    public TimeSpan CriticalNotification { get; }

    protected override IEnumerable<object> EqualityCheckAttributes =>
        new object[] { Configuration, Measurement, Notification, CriticalNotification };
}

and Thresholds:

class Thresholds : ValueObject<Thresholds>
{
    public Thresholds(double notificationLoad, double criticalLoad)
    {
        NotificationLoad = notificationLoad;
        CriticalLoad = criticalLoad;
    }

    public double NotificationLoad { get; }
    public double CriticalLoad { get; }

    protected override IEnumerable<object> EqualityCheckAttributes =>
        new object[] { NotificationLoad, CriticalLoad };
}

Nothing really interesting so far - we just defined equality logic for configurations to help detect changes.

Here are some fake implementations to make test work:

class Notifier : INotifier
{
    public void Notify(bool critical) => Console.WriteLine(critical);
}

class CpuCounter : IPerformanceCounter
{
    public double Value => Math.Sin(DateTime.Now.Millisecond / 1000.0);
}

Fake configuration comes from this class (try to change its content dynamically):

class ConfigurationReader : IConfigurationReader
{
    public Configuration Read() => // fake implementation
        new Configuration(
            new Intervals(
                configuration: TimeSpan.FromSeconds(10),
                measurement: TimeSpan.FromSeconds(0.01),
                notification: TimeSpan.FromSeconds(2),
                criticalNotification: TimeSpan.FromSeconds(1)),
            new Thresholds(
                notificationLoad: 0.3,
                criticalLoad: 0.4));
}

Here comes core RX stuff. I would say that it is really compact and maintainable realization:

class HardwareMonitor : IDisposable
{
    readonly IConfigurationReader ConfigurationReader = new ConfigurationReader();
    readonly IPerformanceCounter PerformanceCounter = new CpuCounter();
    readonly INotifier Notifier = new Notifier();

    public HardwareMonitor()
    {
        Setup(ConfigurationReader.Read());
    }        

    void Setup(Configuration c)
    {
        _sync?.Dispose();
        _notify?.Dispose();

        _sync = Observable
            .Interval(c.Intervals.Configuration)
            .Select(i => ConfigurationReader.Read())
            .DistinctUntilChanged()
            .Subscribe(Setup);

        var alarms = Observable
            .Interval(c.Intervals.Measurement)
            .Select(i => PerformanceCounter.Value)
            .Where(load => load > c.Thresholds.NotificationLoad)
            .Select(load => load > c.Thresholds.CriticalLoad);

        _notify = Observable.Merge(
            alarms 
                .Where(critical => critical)
                .Sample(c.Intervals.Notification), 
            alarms 
                .Where(critical => !critical)
                .Sample(c.Intervals.CriticalNotification)) 
            .Subscribe(Notifier.Notify);
    }

    public void Dispose()
    {
        _sync.Dispose();
        _notify.Dispose();
    }

    IDisposable _sync;
    IDisposable _notify;
}

Feel free to let me know if you have questions.

Answer 2

Here it is - RX example. It might not be exactly what are you looking for but demonstrates "applicability" of the library in your case.

To make it work - create a console app and add RX-Main NuGet package.

Now let's define some infrastructure:

interface IIntervals
{
    TimeSpan Measurement { get; } // how often to measure
    TimeSpan Notification { get; } // notification interval for high load
    TimeSpan CriticalNotification { get; } // notification interval for critical load
}

interface IPerformanceCounter
{
    double Value { get; } // represens cpu load
}

interface IThresholds
{
    double NotificationLoad { get; } // high load threshold
    double CriticalLoad { get; } // critical load threshold
}

interface INotifier
{
    void Notify(bool critical); // sends email
}

Here are some fake implementations - just makes it work somehow.

class Intervals : IIntervals
{
    public TimeSpan Measurement => TimeSpan.FromSeconds(0.01);
    public TimeSpan CriticalNotification => TimeSpan.FromSeconds(1);
    public TimeSpan Notification => TimeSpan.FromSeconds(2);
}

class CpuCounter : IPerformanceCounter
{
    public double Value => Math.Sin(DateTime.Now.Millisecond / 1000.0);
}

class Thresholds : IThresholds
{
    public double CriticalLoad => 0.4;
    public double NotificationLoad => 0.3;
}

class Notifier : INotifier
{
    public void Notify(bool critical) => Console.WriteLine(critical);
}

OK, now the magic:

    static void Main(string[] args)
    {
        IIntervals intervals = new Intervals();
        IPerformanceCounter cpuCounter = new CpuCounter();
        IThresholds thresholds = new Thresholds();
        INotifier notifier = new Notifier();

        var alarms = Observable
            .Interval(intervals.Measurement)  // generate endless sequence of events
            .Select(i => cpuCounter.Value) // convert event index to cpu load value
            .Where(load => load > thresholds.NotificationLoad) // filter out events when load is normal
            .Select(load => load > thresholds.CriticalLoad); // is critical? convert load to boolean

        Observable.Merge(
            alarms // here we throttle critical alarms 
                .Where(critical => critical)
                .Sample(intervals.Notification), // allow critical notification no more often then ...
            alarms // here we throttle non critical alarms
                .Where(critical => !critical)
                .Sample(intervals.CriticalNotification)) // allow non critical notification no more often then ...
            .Subscribe(notifier.Notify); // our actoin to send notifications

        Console.ReadLine();
    }

Answer 3

CpuMonitor definitely looks having a lot of responsibilities. Have you ever consider implementing IObservable<T> here? It is an inverted IEnumerable, "push" instead of "pull", so it works really well for event sources and they filtering/handling in a time sensitive manner. Reactive Extensions (Rx) for .NET provides you with some kind of LINQ for observable sources. There is a free downloadable book Intoduction to Rx - you will just need a part of it to grasp the idea.