3
\$\begingroup\$

I just thought about creating a C# program to download all chapters of a manga given an URL. HTML parsing is done with HtmlAgilityPack.

Issues I have yet to work out are the blocking of the whole program, the rather slow performance of GetPagesLink() as it calls LoadHtmlCode() which uses WebClient a lot (one Webclient object for every page inside a chapter multiplied by number of chapters) and the continuous increase of used memory. At first it starts with ~14 mb but increases infinitely. Besides that, everything works.

using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using System.Windows.Forms;
using Treasure;

namespace MangaRipper
{
    public partial class Form1 : Form
    {
        #region Properties

        private Uri _Uri;

        public Uri Uri
        {
            get { return _Uri; }
            set { _Uri = value; }
        }

        private List<Tuple<string, string, string>> _Chapters = new List<Tuple<string, string, string>>();

        public List<Tuple<string, string, string>> Chapters
        {
            get { return _Chapters; }
            set { _Chapters = value; }
        }

        private string _MangaName;

        public string MangaName
        {
            get { return _MangaName; }
            set { _MangaName = value; }
        }

        #endregion Properties

        public Form1()
        {
            InitializeComponent();
        }

        private void exitToolStripMenuItem_Click(object sender, EventArgs e)
        {
            Application.Exit();
        }

        private string LoadHtmlCode(string url)
        {
            using (WebClient client = new WebClient())
            {
                try
                {
                    // Avoid too many connection requests at once to prevent website from blocking us
                    System.Threading.Thread.Sleep(200);
                    client.Encoding = Encoding.UTF8;
                    client.Proxy = null;
                    return client.DownloadString(url);
                }
                catch (Exception ex)
                {
                    Logger.Log(ex.Message);
                    throw;
                }
            }
        }

        private void btnLoad_Click(object sender, EventArgs e)
        {
            // Multiple mangas are delimited by a semicolon..
            string t = txtURL.Text;
            string[] split = t.Split(';');

            foreach (var item in split)
            {
                CreateDirectory(Path.GetFileNameWithoutExtension(item));
                MangaName = Path.GetFileNameWithoutExtension(item);
                Uri tempUri = new Uri(item);
                Uri = tempUri;
                try
                {
                    using (WebClient client = new WebClient())
                    {
                        string htmlCode = LoadHtmlCode(Uri.AbsoluteUri);
                        LoadAllChapters(htmlCode);
                        Download();
                    }
                }
                catch (Exception ex)
                {
                    Logger.Log(ex.Message);
                    Logger.Log(ex.StackTrace);
                    Logger.Log(ex.InnerException.ToString());
                }
            }
        }

        private void CreateDirectory(string dirName)
        {
            if (!Directory.Exists(dirName))
            {
                Directory.CreateDirectory(dirName);
            }
        }

        private void Download()
        {
            foreach (var chapter in Chapters)
            {
                string bla = chapter.Item2;
                string chapterName = bla.Replace("?", "%3F").Replace(":", "%3A");
                // Skip this chapter if it already exists based on chapter name
                // TODO: Find better way to determine this. Incomplete downloads to a folder would be marked as completed...
                if (Directory.Exists(string.Format("{0}/{1} - {2}", MangaName, chapter.Item3, chapterName)))
                {
                    continue;
                }
                else
                {
                    Directory.CreateDirectory(string.Format("{0}/{1} - {2}", MangaName, chapter.Item3, chapterName));
                }
                List<Tuple<string, int>> temp = new List<Tuple<string, int>>();
                foreach (var item in GetPagesLink(chapter.Item1))
                {
                    temp.Add(new Tuple<string, int>(GetImageLink(item.Item1), item.Item2));
                }

                foreach (var img in temp)
                {
                    WebClient webClient = new WebClient();
                    webClient.DownloadProgressChanged += new DownloadProgressChangedEventHandler(ProgressChanged);
                    webClient.DownloadFileAsync(new Uri(img.Item1), string.Format("{0}/{1} - {2}/{3}.jpg", MangaName, chapter.Item3, chapterName, img.Item2)); // TODO: Find image type and replace hardcoded jpg
                    System.Threading.Thread.Sleep(150);
                }
                temp.Clear();
                txtDebug.AppendText("Finished chapter " + chapter.Item3 + "\r\n");
            }
            Chapters.Clear();
        }

        private void ProgressChanged(object sender, DownloadProgressChangedEventArgs e)
        {
            progressBar1.Value = e.ProgressPercentage;
        }

        /// <summary>
        /// Return a list containing tuples with the direct url to all pages of a chapter
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        private List<Tuple<string, int>> GetPagesLink(string url)
        {
            List<Tuple<string, int>> pages = new List<Tuple<string, int>>();
            HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
            htmlDoc.LoadHtml(LoadHtmlCode(url));
            int counterPage = 1;

            foreach (HtmlNode node in htmlDoc.DocumentNode.SelectNodes("//select[@id='pageMenu']//option"))
            {
                //Console.WriteLine("Value=" + node.Attributes["value"].Value);
                //Console.WriteLine("InnerText=" + node.InnerText);

                pages.Add(new Tuple<string, int>("http://" + Uri.Host + node.Attributes["value"].Value, counterPage));
                counterPage++;
            }
            return pages;
        }

        /// <summary>
        /// Extract direct download link of an image by given url
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        private string GetImageLink(string url)
        {
            HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
            htmlDoc.LoadHtml(LoadHtmlCode(url));

            return htmlDoc.GetElementbyId("img").GetAttributeValue("src", "not found");
        }

        /// <summary>
        /// Load all chapter urls into Chapters property
        /// </summary>
        /// <param name="htmlCode"></param>
        private void LoadAllChapters(string htmlCode)
        {
            HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
            htmlDoc.LoadHtml(htmlCode);

            var chapterLink = htmlDoc.DocumentNode.SelectNodes(@"//div[@id='chapterlist']//a/@href");
            var chapterName = htmlDoc.DocumentNode.SelectNodes(@"//div[@id='chapterlist']//a/@href/following-sibling::text()[1]").Reverse().ToList();
            for (int i = 0; i < chapterLink.Count; i++)
            {
                var link = "http://" + Uri.Host + chapterLink[i].GetAttributeValue("href", "not found");
                var name = chapterName[i].OuterHtml.Replace(" : ", "");
                var number = chapterLink[i].InnerText;
                Chapters.Add(new Tuple<string, string, string>(link, name, number));

                checkedListBox1.Items.Add(link);
            }
        }
    }
}

For logging I wrote a logger in a separate class and namespace:

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace Treasure
{
    /// <summary>
    /// Logger class. Creates per-day logs, saved into "Log" directory
    /// </summary>
    public class Logger
    {
        /// <summary>
        /// Logs the specified log message
        /// </summary>
        /// <param name="LogMsg">The message to be logged.</param>
        /// <param name="args">Arguments like {1}</param>
        public static void Log(string LogMsg, params object[] args)
        {
            if (!Directory.Exists(Environment.CurrentDirectory + @"\Log"))
            {
                try
                {
                    Directory.CreateDirectory(Environment.CurrentDirectory + @"\Log");
                    using (StreamWriter sw = new StreamWriter(Environment.CurrentDirectory + @"\Log\" + DateTime.Now.ToString("dd-MM-yy") + ".txt", true))
                    {
                        sw.WriteLine(DateTime.Now.ToLongTimeString() + " " + string.Format(LogMsg, args));
                    }
                }
                catch (Exception e)
                {
#if DEBUG
                    Console.WriteLine(e.Message + e.StackTrace);
#endif

#if RELEASE
                    MessageBox.Show("Could not create Log directory.", "Error");
                    Log(e.Message);
#endif
                }
            }
            else
            {
                try
                {
                    // Creates per-day log file with current date as file name
                    using (StreamWriter sw = new StreamWriter(Environment.CurrentDirectory + @"\Log\" + DateTime.Now.ToString("dd-MM-yy") + ".txt", true))
                    {
                        sw.WriteLine(DateTime.Now.ToLongTimeString() + " " + string.Format(LogMsg, args));
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.Message + e.StackTrace);
                    Log(e.Message + "\n" + e.StackTrace);
                }
            }
        }
    }
}
\$\endgroup\$
  • 1
    \$\begingroup\$ Suggestions unrelated to performance: do a stylecop pass; remove unused usings; use implicit property backing variables (i.e. public Uri Uri { get; set; }) \$\endgroup\$ – Reinderien May 18 '15 at 21:37
  • 1
    \$\begingroup\$ And don't roll your own logging class. There are plenty of good ones from NuGet. \$\endgroup\$ – Reinderien May 18 '15 at 21:39
  • 1
    \$\begingroup\$ Your Log method is a StackOverflowException waiting to happen. Don't reinvent the wheel anyway. I'd grab NLog from NuGet. \$\endgroup\$ – Zer0 May 18 '15 at 21:40
  • \$\begingroup\$ LoadHtmlCode(string url) should be made to accept an actual Uri instance and not a string (avoid stringly-typed variables). \$\endgroup\$ – Reinderien May 18 '15 at 21:41
  • \$\begingroup\$ @Zer0 I though about using a already existing framework for logging but all the overhead and initial config looks like overkill to me. Of course I will consider it for a large scale project \$\endgroup\$ – Takeru May 18 '15 at 22:30
2
\$\begingroup\$

A large part of the problem is doing Thread.Sleep on the UI thread. You could mark the methods async and use await Task.Delay instead if you want a non-blocking sleep.

public async Task MethodThatRunsOnUIThread()
{
    //Do stuff
    //Wait
    await Task.Delay(150);
    //Do more stuff
}

But if I designed this I would do the entire workload asynchronously. That would involve a little bit more work. But you could start with something simple like this:

  1. Disable part of the UI.
  2. await a Task.Run on the entire workload.
  3. Re-enable part of the UI to allow a second run.

Something like this:

private async void btnLoad_Click(object sender, EventArgs e)
{
    btnLoad.Enabled = false;
    string url = txtURL.Text;
    await Task.Run(() => DoLoad(url));
    btnLoad.Enabled = true;
}

Then while performing work asynchronously you can marshal back to the UI thread to update status using BeginInvoke.

I'm not sure why you're creating a million WebClient instances. Sometimes not even using them:

using (WebClient client = new WebClient())
{
    string htmlCode = LoadHtmlCode(Uri.AbsoluteUri);
    LoadAllChapters(htmlCode);
    Download();
}

There's nothing stopping you from sharing a WebClient instance.

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.