Asynchronous start/stop state transitions

Question

I have a third-party object with asynchronous start and stop methods. Each start and stop may fail with exception. The object is not re-entrant, i.e. I can only call its start or stop method after the previous start/stop has completed.

I need to have a class that handles those transitions to the correct (=last asked) state, while minimizing the number of transitions, allowing my client to submit any number of start/stop requests from any thread at any time.

Currently, I’ve implemented that functionality as endless loop in the async method. However It’s too complex, the loop is over 4 pages long, on each iteration I need to manually switch between 8 states (with the following 3 bits: need to be started/stopped, did tried to start/stop, did failed/succeeded). And the complete class is ~10 pages long.

I have a feeling I might be missing something obvious here. And also that my code looks somewhat similar to what compiler does when compiling an async function. Is there a better way to approach the problem?

using awaiter = TaskCompletionSource<bool>;
using awaiterList = List<TaskCompletionSource<bool>>;

/// <summary>This is the API what we need to have. Looks simple, huh?</summary>
interface iStateMachine
{
    /// <summary>Transition to started state, marshal exceptions to the Task</summary>
    Task startupAsync();

    /// <summary>Transition to started state, fire &amp; forget way, marshal exceptions to the delegate.</summary>
    void startup();

    /// <summary>Transition to stopped state, marshal exceptions to the Task</summary>
    Task shutdownAsync();
    /// <summary>Transition to stopped state, fire &amp; forget way, marshal exceptions to the delegate.</summary>
    void shutdown();
}

class StateMachine: iStateMachine
{
    /// <summary>Initialize</summary>
    /// <param name="startup">Startup implementation</param>
    /// <param name="shutdown">Shutdown implementation</param>
    /// <param name="failed">The delegate to call if startup or shutdown fails while no client is awaiting on the task.</param>
    public StateMachine( Func<Task> startup, Func<Task> shutdown, Action<Exception> failed )
    {
        if( null == startup || null == shutdown )
            throw new ArgumentNullException();
        if( null == failed )
            failed = ( Exception ex ) => { };

        m_startup = startup;
        m_shutdown = shutdown;
        m_failed = failed;
    }

    static void print( string fmt, params object[] args )
    {
        ConsoleEx.print( ConsoleColor.Blue, fmt, args );
    }

    public override string ToString()
    {
        lock ( syncRoot )
        {
            return String.Format( "State = {0}, desired = {1}", m_state, m_bShouldRun );
        }
    }

    // True to perform state transitions in a thread pool thread, false to use the caller's thread
    const bool bUseThreads = false;

    readonly Func<Task> m_startup, m_shutdown;
    readonly Action<Exception> m_failed;

    readonly object syncRoot = new object();

    enum eState : byte
    {
        stopped,
        pending,
        started
    }

    eState m_state = eState.stopped;
    bool m_bShouldRun = false;

    awaiterList m_awStart = new awaiterList( 2 );
    awaiterList m_awStop = new awaiterList( 2 );

    static Task addTask( awaiterList list )
    {
        awaiter res = new awaiter();
        list.Add( res );
        return res.Task;
    }

    static Task completed()
    {
        return Task<bool>.FromResult( true );
    }

    public Task startupAsync()
    {
        Task res = null;
        lock ( syncRoot )
        {
            if( m_state == eState.started )
            {
                print( "Already started" );
                return completed(); // already started
            }

            m_bShouldRun = true;

            res = addTask( m_awStart );
            if( m_state == eState.pending )
            {
                return res; // pending = mainLoop should handle the state transition
            }

            m_state = eState.pending;
        }
        mainLoop( true );
        return res;
    }

    public void startup()
    {
        lock ( syncRoot )
        {
            if( m_state == eState.started )
            {
                print( "Already started" );
                return; // already started
            }

            m_bShouldRun = true;
            if( m_state == eState.pending )
                return; // pending = mainLoop should handle the state transition

            m_state = eState.pending;
        }
        mainLoop( true );
    }

    public Task shutdownAsync()
    {
        Task res = null;
        lock ( syncRoot )
        {
            if( m_state == eState.stopped )
            {
                print( "Already shut down" );
                return completed(); // already stopped
            }

            m_bShouldRun = false;
            res = addTask( m_awStop );
            if( m_state == eState.pending )
                return res; // pending = mainLoop should handle the state transition

            m_state = eState.pending;
        }
        mainLoop( false );
        return res;
    }

    public void shutdown()
    {
        lock ( syncRoot )
        {
            if( m_state == eState.stopped )
            {
                print( "Already shut down" );
                return; // already stopped
            }

            m_bShouldRun = false;
            if( m_state == eState.pending )
                return; // pending = mainLoop should handle the state transition

            m_state = eState.pending;
        }
        mainLoop( false );
    }

    /// <summary>Empty the list, return items in another list.</summary>
    static awaiterList getList( awaiterList src )
    {
        awaiterList res = src.ToList();
        src.Clear();
        return res;
    }

    /// <summary>Main loop that actually changes the state of dat implementation object.</summary>
    async void mainLoop( bool shouldStart )
    {
        bool? wasStarting = null;
        Exception exFailed = null;
        while( true )
        {
            lock ( syncRoot )
            {
                if( wasStarting.HasValue )
                {
                    // Already tried transitioning..
                    int stateMask = 0;
                    if( m_bShouldRun )
                        stateMask |= 1;
                    if( wasStarting.Value )
                        stateMask |= 2;
                    if( null != exFailed )
                        stateMask |= 4;

                    switch( stateMask )
                    {
                        case 0:
                            // Shouldn't run, was stopping, succeeded -> finish in eState.stopped
                            complete( m_awStop );
                            cancel( m_awStart );
                            m_state = eState.stopped;
                            return;

                        case 1:
                            // Should run, was stopping, succeeded -> now start
                            complete( m_awStop );
                            break;

                        case 2:
                            // Shouldn't run, was starting, succeeded -> now stop
                            complete( m_awStart );
                            break;
                        case 3:
                            // Should run, was starting, succeeded -> finish in eState.started
                            complete( m_awStart );
                            cancel( m_awStop );
                            m_state = eState.started;
                            return;

                        case 4:
                            // Shouldn't run, was stopping, failed -> finish in eState.stopped
                            if( !fail( m_awStop, exFailed ) )
                                m_failed( exFailed );
                            cancel( m_awStart );
                            m_state = eState.stopped;
                            return;

                        case 5:
                            // Should run, was stopping, failed -> now start
                            if( !fail( m_awStop, exFailed ) )
                                m_failed( exFailed );
                            break;
                        case 6:
                            // Shouldn't run, was starting, failed -> not sure, but probably finish in eState.stopped
                            if( !fail( m_awStart, exFailed ) )
                                m_failed( exFailed );
                            complete( m_awStop );
                            m_state = eState.stopped;
                            return;

                        case 7:
                            // Should run, was starting, failed
                            if( !fail( m_awStart, exFailed ) )
                                m_failed( exFailed );
                            complete( m_awStop );
                            m_state = eState.stopped;
                            return;
                    } // switch( stateMask )
                } // if( wasStarting.HasValue )
                else
                {
                    // Never tried transitioning
                    if( m_bShouldRun != shouldStart )
                    {
                        // The client already changed the mind between the 2 locks()
                        complete( m_awStart );
                        complete( m_awStop );
                        m_state = m_bShouldRun ? eState.started : eState.stopped;
                        return;
                    }
                }

                shouldStart = m_bShouldRun;
            } // unlock( syncRoot ) 

            // Perform the state transition
            wasStarting = shouldStart;
            try
            {
                Func<Task> fn = shouldStart ? m_startup : m_shutdown;
                if( bUseThreads )
                    await Task.Run( fn );
                else
                    await fn();
            }
            catch( Exception ex )
            {
                exFailed = ex;
            }
        }
    }

    /// <summary>Clear the list of awaiters, run the action on the item[s] that was/were there.</summary>
    /// <returns>How many awaiters were on the list.</returns>
    static int callAwaiters( awaiterList list, Action<awaiter> act )
    {
        foreach( var cs in list )
            act( cs );
        int res = list.Count;
        list.Clear();
        return res;
    }

    /// <summary>Mark the awaiters as completed successfully.</summary>
    static void complete( awaiterList list )
    {
        callAwaiters( list, cs => cs.SetResult( true ) );
    }

    /// <summary>Mark the awaiters as canceled.</summary>
    /// <remarks>AFAIR the TaskCancelledException will be marshaled to the clients.</remarks>
    static void cancel( awaiterList list )
    {
        callAwaiters( list, cs => cs.SetCanceled() );
    }

    /// <summary>Mark the awaiters as failed.</summary>
    /// <param name="list"></param>
    /// <param name="ex">Exception to marshall to the clients</param>
    /// <returns>false if no one had awaited for this fail</returns>
    static bool fail( awaiterList list, Exception ex )
    {
        return callAwaiters( list, cs => cs.SetException( ex ) ) > 0;
    }
}

Here’s the complete demo project. It is just isolated demo; in the real life, the client calls StateMachine from different threads, potentially simultaneously. That’s not a server however, the software is Desktop + Store + Phone, the scalability isn’t a priority: there’re merely 1-2 such objects in the application, and transition takes up to several seconds.

Answer 1

I have written many state-machines in C# (related to hardware-communication) and hopefully my answer can be useful. Re-writing your existing state-machine will take too long, therefore some hints instead.

A state-machine should have the following components:

• An action/signal-queue
• An enumerator variable that describes all possible states for your state-machine.
• An enumerator with all possible signals (I like to compare signals with method-calls/actions)
• A message loop (main loop)
• Methods to post signals to the action/signal queue.
• A collection of timers (for timeout-signals)
• Methods to post timeout-signals.
• Methods to cancel (remove) timer (timeout) signals from the timer collection
• Timer-event handler that posts timeout-signals to the signal queue.
• Methods to start and stop the state machine (loop)

Start the state-machine's on its own thread. This thread's mission is to drive the message-loop:

1. De-queue next signal from the queue (the queue should block the thread if it's empty, but listen to the CancellationToken so that the state-machine can be stopped even if queue is empty)
2. Post the signal to the message loop
3. Break the messsage-loop if the CancellationToken has be signalled
4. Goto 1

I think you're missing the concept of timeout-signals in your state-machine. When you try to start or stop an awaiter, you should post timeout-signals to your state-machine's message-loop that will be posted to your signal queue when the timeout-period expires (and the timeout-signal hasn't been cancelled due to some other signal/event).

In your main loop you switch for current state and current signal

public void MainLoop(MySignalEnum signal)
{
    switch(this.CurrentState) 
    {
        case MyStateMachineStates.Stopped:
            switch(signal)
            {
                 case MySignalEnum.Start:
                    TryToStartDevices();
                    PostTimeout(MySignalEnum.Poll, 100); // Poll device every 100ms
                    PostTimeout(MySignalEnum.TimeoutStart, 1000);
                    // Change to new state-machine state..
                    this.CurrentState = MyStateMachineStates.Starting;
                    break;
            }
            break;
        case MyStateMachineStates.Starting:
            switch(signal) 
            {

                 case MySignalEnum.TimeoutStart:
                    // OH NO Devices didn't start in a timely fashion
                    this.CurrentState = MyStateMachineStates.Error;
                    break;
                 case MySignalEnum.Poll:
                    var deviceStates = ReadDeviceStates();
                    if(deviceStates == DeviceStates.BothStarted)
                    {
                         this.CurrentState = MyStateMachineStates.Started;
                         KillTimeout(MySignalEnum.TimeoutStart);
                         // Dont post poll-timeout signal. No need to poll anymore
                         // KillTimeout will remove the timeout signal started below
                         // Its good practice to kill Timeout-signals when the other signal you've been waiting for has been received. In rare cases you might return to this state before the timeout is posted to the message queue and you will receive a timeout instantiated the last time you entered this state. These errors are very hard to detect!
                    }
                    else
                    {
                         // else wait 100ms and poll device again
                         // note: if this poll-sequence takes longer than 1000ms
                         // the TimeoutStart signal will be posted and set the
                         // state machine in error state
                         PostTimeout(MySignalEnum.Poll, 100);
                    }
            }
    }
}

The advantage is that every signal runs in the order they are posted to the signal queue. Many clients can post signals to the state-machine but they will all be executed in the order they are positioned in the message-queue, one by one, on a single thread. This reduces complexity a lot.

If a signal can't be processed in the state-machine's current state it is just ignored which is allowed (encouraged even). Client's can still read the states-machine's current state to find out what signals it should post at any given time.

Notes on the PostTimeout method: It takes a signal and timeout-value (milliseconds). It creates a Threading.Timer, starts the timer with the timeout-values and raises an event with the signal as a parameter when the timeout expires. The timer's event handler posts the signal (immediatly) to the signal-queue.