Small sitemap checker .aspx file

Question

I wrote a small sitemap-checker.aspx, the goal of this code is to find the pages, that respond with the codes other than 200. I've only put the C# part here for the review:

    void Start_Click(object sender, EventArgs e)
    {
        CheckUrls(UrlTextBox.Text);
    }

    private void CheckUrls(string url)
    {
        try
        {
            var xmlDoc = LoadXml(url);
            var urlNodes = xmlDoc.GetElementsByTagName("url");
            var numberOfNodes = urlNodes.Count;
            var index = 0;
            
            Log("### Total nodes: " + numberOfNodes+ "<br>");
            Log("Here is the list of URLs, with the status code other than 200: <br>");
            foreach (XmlNode urlNode in urlNodes)
            {
                CheckUrlNode(urlNode, xmlDoc);
                index++;

                if(index % 10 == 0)
                {
                    Log("<span style='color:red'>" + index + " nodes of " + numberOfNodes + " are finished, please wait...</span>");
                }
            };
            Log("### Sitemap check finished. Please analyze the reported urls<br><br>");
        }
        catch (Exception e)
        {
            Log("Some problems were encountered while checking the sitemap<br>");
            Log(e.Message + "<br>");
            Log(e.StackTrace + "<br>");
        }
    }

    private XmlDocument LoadXml(string url)
    {
        var m_strFilePath = url;
        string xmlStr;
        using (var wc = new WebClient())
        {
            xmlStr = wc.DownloadString(m_strFilePath);
        }
        var xmlDoc = new XmlDocument();
        xmlDoc.LoadXml(xmlStr);
        return xmlDoc;
    }

    private void CheckUrlNode(XmlNode xmlNode, XmlDocument xmlDoc)
    {
        var urlList = new List<string>() { xmlNode["loc"].InnerText };

        var nsmgr = new XmlNamespaceManager(xmlDoc.NameTable);
        nsmgr.AddNamespace("xhtml", "http://www.w3.org/1999/xhtml");

        var hrefLangUrls = xmlNode
            .SelectNodes("xhtml:link", nsmgr)
            .Cast<XmlNode>()
            .Select(x => x.Attributes["href"].Value);

        urlList.AddRange(hrefLangUrls);

        foreach (var url in urlList)
        {
            string logMessage;
            if (!CheckUrl(url, out logMessage))
            {
                Log(logMessage);
            }
        }
    }

    private bool CheckUrl(string url, out string logMessage)
    {
        logMessage = null;
        try
        {
            var request = HttpWebRequest.Create(url) as HttpWebRequest;
            request.Timeout = 5000; //set the timeout to 5 seconds to keep the user from waiting too long for the page to load
            request.Method = "HEAD"; //Get only the header information -- no need to download any content

            using (var response = request.GetResponse() as HttpWebResponse)
            {
                int statusCode = (int)response.StatusCode;
                if (statusCode == 200) //Good requests
                {
                    return true;
                }
                else
                {
                    logMessage = String.Format("<a href='{0}'>{0}</a>, status code: {1}", url, statusCode);
                    return false;
                }
            }
        }
        catch (WebException ex)
        {
            logMessage = String.Format("<a href='{0}'>{0}</a>, status: {1}", url, ex.Status);
        }
        catch (Exception ex)
        {
            logMessage = String.Format("Could not test url {0}, exeption message: {1}", url, ex.Message);
        }

        return false;
    }

Answer 1

I think you can achieve the same with less code by utilizing some libraries and language features.

Google's Sitemap Protocol is not supported by default in the .NET Framework / Core.
But there are couple of packages which defines the corresponding C# classes, for example:

• Google-Sitemap
• X.Web.Sitemap

I have chosen the second one. So, the parsing logic would look like this:

public static async Task<XDocument> LoadSitemap(string filePath = "./sitemap.xml")
{
    var sitemap = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None);
    return await XDocument.LoadAsync(sitemap, LoadOptions.None, CancellationToken.None);
}

public static Sitemap ParseSitemap(XDocument document)
{
    var serializer = new XmlSerializer(typeof(Sitemap));
    return (Sitemap) serializer.Deserialize(document.CreateReader());
}

The HEAD retrieval could be done by the following helper method:

private static readonly HttpClient client = new HttpClient();

private static async Task<string> GetHead(string location, 
    Func<HttpResponseMessage, string> successHandler, 
    Func<Exception, string, string> failureHandler)
{
    try
    {
        var timeoutPolicy = new CancellationTokenSource(5000);
        var request = new HttpRequestMessage(HttpMethod.Head, location);
        var response = await client.SendAsync(request, timeoutPolicy.Token);
        return successHandler(response);
    }
    catch (Exception ex)
    {
        return failureHandler(ex, location);
    }
}

The success and failure handlers can be implemented like this:

private static string HandleResponse(HttpResponseMessage response)
{
    var url = response.RequestMessage.RequestUri;
    return response.StatusCode == HttpStatusCode.OK 
        ? $"{url} - OK" 
        : $"{url} - Status code: {response.StatusCode}";
}

private static string HandleError(Exception exception, string url)
{
    return exception switch
    { 
        HttpRequestException hre => $"{url} - unreachable due to {hre.Message}",
        TaskCanceledException tce => $"{url} - timed out due to {tce.Message}",
        _ => $"{url} - failed due to {exception}"
    };
}

Let's put all these pieces together:

static async Task Main(string[] args)
{
    var siteMapXml = await LoadSitemap();
    var siteMap = ParseSitemap(siteMapXml);

    var siteChecks = siteMap
        .Select(url => GetHead(url.Location, HandleResponse, HandleError))
        .ToList();

    await Task.WhenAll(siteChecks);

    foreach (var siteCheck in siteChecks)
    {
        Console.WriteLine(await siteCheck);   
    }

    Console.WriteLine("Finished");
}