I just thought about creating a C# program to download all chapters of a manga given an URL. HTML parsing is done with HtmlAgilityPack
.
Issues I have yet to work out are the blocking of the whole program, the rather slow performance of GetPagesLink()
as it calls LoadHtmlCode()
which uses WebClient
a lot (one Webclient object for every page inside a chapter multiplied by number of chapters) and the continuous increase of used memory. At first it starts with ~14 mb but increases infinitely. Besides that, everything works.
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using System.Windows.Forms;
using Treasure;
namespace MangaRipper
{
public partial class Form1 : Form
{
#region Properties
private Uri _Uri;
public Uri Uri
{
get { return _Uri; }
set { _Uri = value; }
}
private List<Tuple<string, string, string>> _Chapters = new List<Tuple<string, string, string>>();
public List<Tuple<string, string, string>> Chapters
{
get { return _Chapters; }
set { _Chapters = value; }
}
private string _MangaName;
public string MangaName
{
get { return _MangaName; }
set { _MangaName = value; }
}
#endregion Properties
public Form1()
{
InitializeComponent();
}
private void exitToolStripMenuItem_Click(object sender, EventArgs e)
{
Application.Exit();
}
private string LoadHtmlCode(string url)
{
using (WebClient client = new WebClient())
{
try
{
// Avoid too many connection requests at once to prevent website from blocking us
System.Threading.Thread.Sleep(200);
client.Encoding = Encoding.UTF8;
client.Proxy = null;
return client.DownloadString(url);
}
catch (Exception ex)
{
Logger.Log(ex.Message);
throw;
}
}
}
private void btnLoad_Click(object sender, EventArgs e)
{
// Multiple mangas are delimited by a semicolon..
string t = txtURL.Text;
string[] split = t.Split(';');
foreach (var item in split)
{
CreateDirectory(Path.GetFileNameWithoutExtension(item));
MangaName = Path.GetFileNameWithoutExtension(item);
Uri tempUri = new Uri(item);
Uri = tempUri;
try
{
using (WebClient client = new WebClient())
{
string htmlCode = LoadHtmlCode(Uri.AbsoluteUri);
LoadAllChapters(htmlCode);
Download();
}
}
catch (Exception ex)
{
Logger.Log(ex.Message);
Logger.Log(ex.StackTrace);
Logger.Log(ex.InnerException.ToString());
}
}
}
private void CreateDirectory(string dirName)
{
if (!Directory.Exists(dirName))
{
Directory.CreateDirectory(dirName);
}
}
private void Download()
{
foreach (var chapter in Chapters)
{
string bla = chapter.Item2;
string chapterName = bla.Replace("?", "%3F").Replace(":", "%3A");
// Skip this chapter if it already exists based on chapter name
// TODO: Find better way to determine this. Incomplete downloads to a folder would be marked as completed...
if (Directory.Exists(string.Format("{0}/{1} - {2}", MangaName, chapter.Item3, chapterName)))
{
continue;
}
else
{
Directory.CreateDirectory(string.Format("{0}/{1} - {2}", MangaName, chapter.Item3, chapterName));
}
List<Tuple<string, int>> temp = new List<Tuple<string, int>>();
foreach (var item in GetPagesLink(chapter.Item1))
{
temp.Add(new Tuple<string, int>(GetImageLink(item.Item1), item.Item2));
}
foreach (var img in temp)
{
WebClient webClient = new WebClient();
webClient.DownloadProgressChanged += new DownloadProgressChangedEventHandler(ProgressChanged);
webClient.DownloadFileAsync(new Uri(img.Item1), string.Format("{0}/{1} - {2}/{3}.jpg", MangaName, chapter.Item3, chapterName, img.Item2)); // TODO: Find image type and replace hardcoded jpg
System.Threading.Thread.Sleep(150);
}
temp.Clear();
txtDebug.AppendText("Finished chapter " + chapter.Item3 + "\r\n");
}
Chapters.Clear();
}
private void ProgressChanged(object sender, DownloadProgressChangedEventArgs e)
{
progressBar1.Value = e.ProgressPercentage;
}
/// <summary>
/// Return a list containing tuples with the direct url to all pages of a chapter
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
private List<Tuple<string, int>> GetPagesLink(string url)
{
List<Tuple<string, int>> pages = new List<Tuple<string, int>>();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(LoadHtmlCode(url));
int counterPage = 1;
foreach (HtmlNode node in htmlDoc.DocumentNode.SelectNodes("//select[@id='pageMenu']//option"))
{
//Console.WriteLine("Value=" + node.Attributes["value"].Value);
//Console.WriteLine("InnerText=" + node.InnerText);
pages.Add(new Tuple<string, int>("http://" + Uri.Host + node.Attributes["value"].Value, counterPage));
counterPage++;
}
return pages;
}
/// <summary>
/// Extract direct download link of an image by given url
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
private string GetImageLink(string url)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(LoadHtmlCode(url));
return htmlDoc.GetElementbyId("img").GetAttributeValue("src", "not found");
}
/// <summary>
/// Load all chapter urls into Chapters property
/// </summary>
/// <param name="htmlCode"></param>
private void LoadAllChapters(string htmlCode)
{
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(htmlCode);
var chapterLink = htmlDoc.DocumentNode.SelectNodes(@"//div[@id='chapterlist']//a/@href");
var chapterName = htmlDoc.DocumentNode.SelectNodes(@"//div[@id='chapterlist']//a/@href/following-sibling::text()[1]").Reverse().ToList();
for (int i = 0; i < chapterLink.Count; i++)
{
var link = "http://" + Uri.Host + chapterLink[i].GetAttributeValue("href", "not found");
var name = chapterName[i].OuterHtml.Replace(" : ", "");
var number = chapterLink[i].InnerText;
Chapters.Add(new Tuple<string, string, string>(link, name, number));
checkedListBox1.Items.Add(link);
}
}
}
}
For logging I wrote a logger in a separate class and namespace:
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
namespace Treasure
{
/// <summary>
/// Logger class. Creates per-day logs, saved into "Log" directory
/// </summary>
public class Logger
{
/// <summary>
/// Logs the specified log message
/// </summary>
/// <param name="LogMsg">The message to be logged.</param>
/// <param name="args">Arguments like {1}</param>
public static void Log(string LogMsg, params object[] args)
{
if (!Directory.Exists(Environment.CurrentDirectory + @"\Log"))
{
try
{
Directory.CreateDirectory(Environment.CurrentDirectory + @"\Log");
using (StreamWriter sw = new StreamWriter(Environment.CurrentDirectory + @"\Log\" + DateTime.Now.ToString("dd-MM-yy") + ".txt", true))
{
sw.WriteLine(DateTime.Now.ToLongTimeString() + " " + string.Format(LogMsg, args));
}
}
catch (Exception e)
{
#if DEBUG
Console.WriteLine(e.Message + e.StackTrace);
#endif
#if RELEASE
MessageBox.Show("Could not create Log directory.", "Error");
Log(e.Message);
#endif
}
}
else
{
try
{
// Creates per-day log file with current date as file name
using (StreamWriter sw = new StreamWriter(Environment.CurrentDirectory + @"\Log\" + DateTime.Now.ToString("dd-MM-yy") + ".txt", true))
{
sw.WriteLine(DateTime.Now.ToLongTimeString() + " " + string.Format(LogMsg, args));
}
}
catch (Exception e)
{
Console.WriteLine(e.Message + e.StackTrace);
Log(e.Message + "\n" + e.StackTrace);
}
}
}
}
}
public Uri Uri { get; set; }
) \$\endgroup\$Log
method is aStackOverflowException
waiting to happen. Don't reinvent the wheel anyway. I'd grab NLog from NuGet. \$\endgroup\$LoadHtmlCode(string url)
should be made to accept an actualUri
instance and not a string (avoid stringly-typed variables). \$\endgroup\$