2
\$\begingroup\$

I listen to a podcast called Crap from the Past. It's a radio show that airs weekly in Minneapolis and specializes in playing music from my childhood. The show has been on the air for ~26 years. I sometimes want to know if they've ever played a particular song, but searching involves going through one of 26 static web pages. (Or, you know, asking Google.) I decided to write my first real Swift program to scrape their pages and put the shows into a CSV file I could look at in Numbers.

//
//  main.swift
//  CrapScraper
//

import Foundation

func getPage(address: URL) -> String {
    let ephemeralConfiguration  = URLSessionConfiguration.ephemeral
    let ephemeralSession = URLSession(configuration: ephemeralConfiguration, delegate:nil, delegateQueue:nil)
    var done = false;
    var result = ""

    let task = ephemeralSession.dataTask(with: address) { (data, response, error) in
        if let error = error {
            print ("error: \(error)")
        }
        else if let data = data,
            let string = String(data: data, encoding: .utf8) {
            result = string
        }
        done = true;
    }
    task.resume()

    // Busy wait until we get a result
    while (!done){
    }

    return result
}

func escapeString(str : inout String)
{
    // Because Numbers and Excel don't actually handle all of RFC4180, we can't just
    // put quotes around the string and escape the inner quotes to handle commas. We
    // need to actually replace them. This changes the data which is not great, but
    // we don't have a lot of choice here.
    str = str.replacingOccurrences(of: ",", with: ";", options: .regularExpression)
}


func extractDate(scanner: Scanner) -> String
{
    var urlPrefix = "http://www.archive.org/details/cftp-"
    let scannerStart = scanner.scanLocation
    var nsScannedData : NSString?
    if !scanner.scanUpTo(urlPrefix, into: &nsScannedData) {
        return ""
    }

    if scanner.isAtEnd {
        scanner.scanLocation = scannerStart
        let secureURLPrefix = "https://www.archive.org/details/cftp-"
        urlPrefix = secureURLPrefix
        if !scanner.scanUpTo(urlPrefix, into: &nsScannedData) {
            return ""
        }
    }
    if !scanner.scanString(urlPrefix, into: nil) {
        return ""
    }

    var dateStr = ""
    var nsDateStr : NSString?
    if !scanner.scanUpTo("\"", into: &nsDateStr) {
        return ""
    }

    dateStr = nsDateStr!.substring(from: 0)

    escapeString(str: &dateStr)

    return dateStr
}

func extractShowName(scanner: Scanner) -> String
{
    // The show name may be blank in some cases
    // If there is a title, it usually starts with ": " which
    // we want to remove
    let anchorEnd = "</a>"
    if !scanner.scanUpTo(anchorEnd, into: nil) {
        return ""
    }

    if !scanner.scanString(anchorEnd, into: nil) {
        return ""
    }

    let headingEnd = "</h2>"
    var nsShowName : NSString?
    if !scanner.scanUpTo(headingEnd, into: &nsShowName) {
        return ""
    }

    var showName = nsShowName!.substring(from: 0)
    if showName.starts(with: ": ") {
        showName.removeFirst(2)
    }
    escapeString(str: &showName)

    return showName;
}

func extractSongs(scanner: Scanner) -> Array<String>
{
    var songs : Array<String> = []
    let indent = "<p class=\"indent\">"
    if !scanner.scanUpTo(indent, into: nil) {
        return songs
    }

    if !scanner.scanString(indent, into: nil) {
        return songs
    }

    var nsSongInfo : NSString?
    let paragraphEnd = "</p>"
    if !scanner.scanUpTo(paragraphEnd, into: &nsSongInfo) {
        return songs
    }

    if !scanner.scanString(paragraphEnd, into: nil) {
        return songs
    }

    let songInfo = nsSongInfo!.substring(from: 0)
    let songScanner = Scanner(string: songInfo)
    let breakTag = "<br>"
    while !songScanner.isAtEnd {
        var nsNextSong : NSString?
        if !songScanner.scanUpTo(breakTag, into: &nsNextSong) {
            return songs
        }

        var nextSong = nsNextSong!.substring(from: 0)
        escapeString(str: &nextSong)
        songs.append(nextSong)

        if !songScanner.scanString(breakTag, into: nil) {
            return songs
        }
    }

    return songs
}

func analyzePage(page: String) -> String {
    let scanner = Scanner(string: page)

    // Sanity check - make sure we've at least got an HTML body tag
    let body  = "<body>"

    if !scanner.scanUpTo(body, into: nil) {
        return ""
    }

    if !scanner.scanString(body, into: nil) {
        return ""
    }

    // Start scanning for the <h2> sections
    let h2  = "<h2>"
    var csv = ""
    while (scanner.scanUpTo(h2, into: nil)) {
        // Read past the <h2>. Note that at the end, scanUpTo() will
        // return true if it reads to the end of the file and then
        // scanString() will fail.
        if !scanner.scanString(h2, into: nil) {
            if scanner.isAtEnd {
                return csv
            }

            return ""
        }

        let dateStr = extractDate(scanner: scanner)

        let showName = extractShowName(scanner: scanner)

        let songs = extractSongs(scanner: scanner)

        let songList = songs.joined(separator: ", ")

        csv.append(dateStr)
        csv.append(", ")
        csv.append(showName)
        csv.append(", ")
        csv.append(songList)
        csv.append("\n")
    }

    return csv
}

var year : Int
print ("\"date\", \"Show Name\", \"Song Titles\"\n")
for year in 1992...2018 {
    let nextAddress = URL(string: "http://crapfromthepast.com/playlists/\(year).htm")
    let nextPage = getPage(address: nextAddress!)
    if (nextPage != "") {
        let analyzedPage = analyzePage(page: nextPage)
        if analyzedPage != "" {
            print ("\(analyzedPage)")
        }
    }
}

Please note that running this will actually grab the pages from the website, so please be respectful and don't hammer them!

\$\endgroup\$
  • \$\begingroup\$ Instead reinventing the wheel better use already existing nokogiri ports like kanna ( github.com/tid-kijyun/Kanna ) or Fuzi ( github.com/cezheng/Fuzi ) \$\endgroup\$ – muescha Feb 12 '18 at 13:59
  • \$\begingroup\$ Neat! Didn't know those existed. I'll look into that. Thanks! \$\endgroup\$ – user1118321 Feb 12 '18 at 16:57
2
\$\begingroup\$

I'll cover just one aspect of your program: How a series of network requests is handled.

This “polling”

// Busy wait until we get a result
while (!done){
}

is bad because it wastes CPU cycles. In my test it caused almost 100% usage for one CPU core while a network request is active.

Network requests are asynchronous in nature, and there are better (and less resource intensive) ways to handle that.

First change the getPage() function to take a callback instead of returning the result. It is also sufficient to create the URLSession once:

let ephemeralConfiguration = URLSessionConfiguration.ephemeral
let ephemeralSession = URLSession(configuration: ephemeralConfiguration,
                                  delegate: nil, delegateQueue: nil)

func getPage(address: URL, callback: @escaping (_ page: String) -> Void) {

    let task = ephemeralSession.dataTask(with: address) { (data, response, error) in
        if let error = error {
            print("error: \(error)")
            callback("")
        } else if let data = data,
            let string = String(data: data, encoding: .utf8) {
            callback(string)
        } else {
            print("Bad data")
            callback("")
        }
    }
    task.resume()
}

One possible approach would now be to start the first request, and in the completion handler start the next request, until all pages have been retrieved.

Another approach is to use a “dispatch group” to chain the network requests. This allows to almost keep the structure of the main loop:

let group = DispatchGroup()
for year in 1992...2018 {
    group.enter()
    let nextAddress = URL(string: "http://crapfromthepast.com/playlists/\(year).htm")!
    getPage(address: nextAddress) { (page) in

        // ... Analyze and print page ... 

        group.leave()
    }
    group.wait()
}

The dispatch group has a “counter” which is increased by group.enter() and decreased by group.leave(). group.wait() waits (with almost no CPU usage) until the counter reaches zero. In our case this mechanism is used to start the next page request only after the previous one has completed.

Another option is to use a “semaphore”:

let sema = DispatchSemaphore(value: 0)
for year in 1992...2018 {
    let nextAddress = URL(string: "http://crapfromthepast.com/playlists/\(year).htm")!
    getPage(address: nextAddress) { (page) in

        // ... Analyze and print page ... 

        sema.signal()
    }
    sema.wait()
}

Here sema.signal() increases the value of the semaphore, making it positive, which is what sema.wait() waits for.

\$\endgroup\$
  • \$\begingroup\$ Thanks so much! I had the problem initially that my block never got executed because the app quit before the network connection completed, so I added a busy-wait knowing full well it was not a good solution. But I didn't know about DisptachGroup(). That's very helpful. \$\endgroup\$ – user1118321 Feb 13 '18 at 1:39
  • \$\begingroup\$ @user1118321: Your app quit early because it had no "run loop". For that is would be sufficient to call CFRunloopRun() or dispatchMain(). A dispatch group (or alternatively, a semaphore) is just easier to use when chaining a series of requests. \$\endgroup\$ – Martin R Feb 13 '18 at 5:03
  • \$\begingroup\$ I think you block is executed before. (Just debug it) but you can not use to get the value via an Return or assigning a value to a global variable in the Block. It should bei more async \$\endgroup\$ – muescha Feb 17 '18 at 9:52

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.