F# thumbnail Scraper

Question

Thumbnail Scraper

I'm a C# programmer learning F#. For my first program I created a console youtube thumbnail scraper. It parses command line arguments with argu and then downloads the thumbnails, displaying download progress with spectre console.

Usage

Example arguments: --channel-name (chanel name) --save-path (valid path) --after-date (optional date filter) --api-key (google api key) -d (optional argument to print some example images)

Code

open System
open Google.Apis.Services
open System.IO
open Google.Apis.YouTube.v3
open System.Text.RegularExpressions
open FSharp.Control
open Argu
open Spectre.Console
open System.Threading.Tasks
open System.Net.Http
open System.Diagnostics

type DownloadersArgs = 
        |[<Mandatory>] Save_Path of path:string
        |[<Mandatory>] Channel_Name of channel:string
        |[<Mandatory>] Api_Key of api_key:string 
        | After_Date of from_date:string 
        | [<AltCommandLine("-d")>] DisplaySample

        interface IArgParserTemplate with
                member s.Usage =
                    match s with
                    | Save_Path _ -> "specify the save location."
                    | Channel_Name _ -> "specify the youtube channel to scrape."
                    | After_Date _ -> "specify the date from which uploads will be collected."
                    | Api_Key _ -> "specify the google api key."
                    | DisplaySample _ -> "print a selection of downloaded images."

let parseArgs arg = 
    let parser = ArgumentParser.Create<DownloadersArgs>()
    let results = parser.Parse(inputs = arg) 
        
    let channelName = results.GetResult Channel_Name
    let path = results.GetResult Save_Path
    let apiKey = results.GetResult Api_Key
        
    let printSamples = match results.TryGetResult DisplaySample with 
                        | Some DisplaySample -> true
                        | _ -> false
    
    let fromDate = results.GetResult(After_Date, defaultValue = "12/1/2000 9:00:00 AM") |> DateTime.Parse 
    (channelName, path, apiKey, fromDate, printSamples)

type VideoData = {Title:string; Url:string; Upload: DateTime option}


let getFileExtension url = 
    let uri = System.Uri(url)
    uri.GetLeftPart(UriPartial.Path) |> Path.GetExtension
    
let createPath folder title url = 
    let fileName = Regex.Replace(title, @"[\\/:*?""<>|]", "")
    let fileExtension = getFileExtension url

    Path.Combine(folder, $"{fileName}{fileExtension}")
    
let downloadImage videoData folder (client:HttpClient) = 
    async {
        let path = createPath folder videoData.Title videoData.Url
        let! imageBytes = client.GetByteArrayAsync(videoData.Url) |> Async.AwaitTask
        do! File.WriteAllBytesAsync(path, imageBytes) |> Async.AwaitTask
        return (videoData, path)
    }
    
let downloadAllImages (images: AsyncSeq<VideoData>) folderPath = 
    Directory.CreateDirectory(folderPath) |> ignore
    let client = new HttpClient()
    
    images |> AsyncSeq.mapAsyncParallel (fun videoData -> downloadImage videoData folderPath client)


let getVideoPage (service:YouTubeService) playlistId pageToken = 
    async {
        if pageToken = null then   
            return None
        else
            let mutable request = service.PlaylistItems.List("snippet")
            
            request.PlaylistId <- playlistId
            request.PageToken <- pageToken
            request.MaxResults <- 50L

            let! response = request.ExecuteAsync() |> Async.AwaitTask

            let pageData = response.Items :> seq<_> 
            let nextToken= response.NextPageToken

            return Some (pageData, nextToken)
            }

let getAllVideoPages service playlistId = 
    let rec loop token = asyncSeq {
        let! result = getVideoPage service playlistId token 

        match result with 
        | Some (pages, nextToken) ->
            for page in pages do
                yield page

            yield! loop nextToken
        | _ -> ()
    }

    loop ""

let getChannelUploadsPlaylistId (service:YouTubeService) channelName = async{
    let mutable channelsListRequest:ChannelsResource.ListRequest = service.Channels.List("contentDetails")
    channelsListRequest.ForUsername <- channelName;

    let! response =  channelsListRequest.ExecuteAsync() |> Async.AwaitTask

    let channel = response.Items :> seq<_> |> Seq.head
    let playlistId = channel.ContentDetails.RelatedPlaylists.Uploads
    return playlistId
}

let getIdAndService channelName apiKey= async {
    let baseClient = new BaseClientService.Initializer(ApiKey = apiKey, ApplicationName = "ThumbnailDownloader") 
    let service = new YouTubeService(baseClient)

    let! playlistId = getChannelUploadsPlaylistId service channelName 
    return (service, playlistId)
}

type StackProcessorMessage<'T> = Send of 'T | Get of AsyncReplyChannel<'T option * int>

// Holds last value it recieved and maintains a count of how many values received
let stackProcessor<'T> = 
    MailboxProcessor.Start(fun inbox ->
        let rec loop (latestState:'T option) counter = 
            async{
                let! msg = inbox.Receive()
                match msg with  
                | Send newState -> return! loop (Some newState) (counter + 1)
                | Get channel -> 
                    channel.Reply(latestState, counter) 
                    return! loop latestState counter
            }
        loop None 0
        )

let createProgressTable = 
    let table = Table().HeavyBorder()
                    .AddColumn("[bold #f1fa8c]Process[/]") 
                    .AddColumn("[bold #f1fa8c]Title[/]") 
                    .AddColumn("[bold #f1fa8c]Uploaded[/]") 
                    .AddColumn("[bold #f1fa8c]Count[/]")
                    .AddRow("Most recently retrieved:","","","0")
                    .AddRow("Most recently downloaded:","","","0/0");

    table.Columns[1].Width <- 50
    table.Columns[2].Width <- 12
    table

let printProgressAsync (downloadTask:Task) (retrievedDataChannel: MailboxProcessor<StackProcessorMessage<_>>) (downloadedChannel:MailboxProcessor<StackProcessorMessage<_>>) = 
    let table = createProgressTable

    let progressTask = AnsiConsole.Live(table).StartAsync((fun ctx -> 
        task{
            let timeSpan = TimeSpan.FromMilliseconds(100)

            let rec loop () = async{
                do! Task.Delay(timeSpan)|> Async.AwaitTask |> Async.Ignore

                let retrievedData, retrievedCount = retrievedDataChannel.PostAndReply(fun rc -> Get rc)
                let downloadedData, downloadedCount = downloadedChannel.PostAndReply(fun rc -> Get rc)

                let getUploadDate (date: DateTime option) = 
                    match date with 
                    | Some date -> date.ToString("dd/M/yyyy")
                    | None -> "N/A"
                       
                do match retrievedData with 
                    | Some videoData -> 
                        table.UpdateCell(0,1,$"{videoData.Title}")
                            .UpdateCell(0,2,$"{getUploadDate videoData.Upload}")
                            .UpdateCell(0,3,$"{retrievedCount}") |> ignore
                    | _-> ()          
                                      
                do match downloadedData with 
                    | Some (videoData,_) -> 
                        table.UpdateCell(1,1,$"{videoData.Title}")
                            .UpdateCell(1,2,$"{getUploadDate videoData.Upload}")
                            .UpdateCell(1,3,$"{downloadedCount}/{retrievedCount}") |> ignore
                    | _-> ()     
                                          
                ctx.Refresh()

                if not downloadTask.IsCompleted then
                    do! loop () 
                else 
                    ()
            }

            do! loop ()
        }))
    progressTask

// Prettier to get the middle image
let getMiddleImage (collection: _ list) =
    let midpoint = collection.Length/2
    let _, (path:string) = collection[midpoint]
    CanvasImage(path)

let printSampleGrid data =
    let images = data |> List.splitInto 3
                |> List.map getMiddleImage
    let sampleGrid = Grid().AddColumns(3).AddRow(images[0],images[1],images[2])
    AnsiConsole.Write(sampleGrid)

let playlistItemToVideoData (item:Data.PlaylistItem ) =
    let nullUpload = item.Snippet.PublishedAt
    let upload = if nullUpload.HasValue then Some nullUpload.Value else None
    {Title=item.Snippet.Title.EscapeMarkup(); Url=item.Snippet.Thumbnails.Default__.Url; Upload= upload}


[<EntryPoint>]
let main argv =
    try 
        let channelName, path, apiKey, fromDate, displayImages = parseArgs argv

        let stopWatch = Stopwatch.StartNew()

        AnsiConsole.MarkupLine($"Downloading thumbnails from channel [bold #8be9fd]{channelName.EscapeMarkup()}[/] to path [bold #bd93f9]{path}[/]")
        let service, channelId = getIdAndService channelName apiKey |> Async.RunSynchronously

        AnsiConsole.MarkupLine($"Retrieved channel id [bold #ff79c6]{channelId}[/] for channel [bold #8be9fd]{channelName}[/] \n")

        let retrievedDataChannel = stackProcessor
        let downloadedChannel = stackProcessor

        let isAfterDate vid = match vid.Upload with
                              | Some value -> value > fromDate;
                              | _-> false
        
        let downloadTask =  (service, channelId) ||> getAllVideoPages 
                            |> AsyncSeq.map playlistItemToVideoData 
                            |> AsyncSeq.takeWhile isAfterDate 
                            |> AsyncSeq.map (fun x-> retrievedDataChannel.Post(Send x) |> ignore; x)
                            |> (fun images -> downloadAllImages images path) 
                            |> AsyncSeq.map (fun x-> downloadedChannel.Post(Send x) |> ignore; x)
                            |> AsyncSeq.toListAsync
                            |> Async.StartAsTask

        let printProgressTask = printProgressAsync downloadTask retrievedDataChannel downloadedChannel

        let result = downloadTask |> Async.AwaitTask |> Async.RunSynchronously
        printProgressTask |> Async.AwaitTask |> Async.RunSynchronously
        let _, downloadedCount = downloadedChannel.PostAndReply(fun rc -> Get rc)

        if displayImages then 
            printSampleGrid result

        let elapsedTime = (int)stopWatch.Elapsed.TotalMilliseconds
        AnsiConsole.MarkupLine($"Downloaded [bold #ff5555]{downloadedCount}[/] items from [bold #8be9fd]{channelName}[/] "
        + $"to path [bold #bd93f9]{path}[/] in [bold white]{elapsedTime}ms[/]") 
        
    with e -> 
        AnsiConsole.MarkupLine($"[bold #ff5555]Error occured[/]\r\n[bold #ffb86c]{e.Message.EscapeMarkup()}[/]\r\n[bold #f1fa8c]{e.StackTrace.EscapeMarkup()}[/]")
        raise e
    0

Aside from usual questions about general correctness and idiomaticy I have a couple of questions.

• Is the asynchronous code correct, F#'s cold tasks seem very different to C#.
• Could Reactive.Observable be used instead of AsyncSeq after the TakeWhile section to speed up downloads.
• I originally used modules to group my functions but removed them after reading more F# code. Was this the right decision? Should local functions be used.
• Is splitting up functions preferred? Ie getFileExtension and createPath could be inlined.
• Was MailboxProcessor the correct choice for updating the console?