In this project I'm working on, I need to extract articles text from their original HTML document. This class, HtmlConnection
, receives the URL of the article, and eventually it needs to produce a collection of the paragraphs inside the article. I'm using HTML agility pack and XPath to extract only relevent text from the article, "removing" irrelevant text that comes alongside in the HTML, such as JavaScript, etc. Notice that this class does not produce the final text of the article (another class deals with it), but rather a HtmlNodeCollection
that consist of all the paragraphs in the article.
There is one main issue in the code: It's too slow.
I did some test, and came up with these numbers:
- Number of articles downloaded: 25
- Average download time: 4958 Miliseconds
This is too much. As you can see, just 25 articles take about 2 minutes. And I plan to download hundreds of articles per run. It could be a problem with my internet connection, but when I'm surfing normally, it's pretty fast and clean.
Imports HtmlAgilityPack
Imports System.Text.RegularExpressions
Imports System.Net
''' <summary>
''' Represents a single Html document.
''' </summary>
Public Class HtmlConnection
' XPath for all the paragraphs inside the body.
Private Const BodyPath As String = "//body//p"
' RegEx for a single word.
Private Const WordPath As String = "[a-zA-Z]+"
''' <summary>
''' Constructor to initialize Url property
''' and to call the DownloadHtml sub.
''' </summary>
''' <param name="url">
''' The Url of the current article.
''' </param>
Public Sub New(ByVal url As String)
Me.Url = url
DownloadHtml()
End Sub ' Constructor
''' <summary>
''' Represents the Url of the current article.
''' </summary>
Private Property Url As String
''' <summary>
''' Represents all the Html code
''' received from the article.
''' </summary>
Private Property FullHtml As HtmlDocument
Private _BodyHtml As HtmlNodeCollection
''' <summary>
''' Represents the Html of all the paragraph inside the body.
''' </summary>
Public Property BodyHtml As HtmlNodeCollection
Get
Return _BodyHtml
End Get
Set(value As HtmlNodeCollection)
Dim WordsMatches As MatchCollection
_BodyHtml = value
' Iterate through all the paragraphs in order to
' count the number of words in them. We assume that a
' paragraph should be more then 10 words at least in order
' to be considered as part of the article, and not as an
' irrelevant text, such as the name of the author or a date,
' which are usually presented in an independent paragraph.
' We operate in a descending order to prevent wrong
' filtration or an "index was out of range" error.
For Paragraph As Integer = value.Count - 1 To 0 Step -1
WordsMatches = Regex.Matches(value.Item(Paragraph).InnerText, WordPath)
If WordsMatches.Count < 10 Then
_BodyHtml.RemoveAt(Paragraph)
End If
Next
End Set
End Property
''' <summary>
''' Creates a new Html DOM using XPath.
''' </summary>
Private Sub DownloadHtml()
' HtmlWeb uses Http protocol to download
' Html documents according to a certain Url.
Dim HtmlWeb As HtmlWeb = New HtmlWeb
FullHtml = New HtmlDocument
' Because BodyHtml is a collection, it needs to be
' initialize. Thus, we create a new HtmlNodeColleciton
' that does not actually possess any nodes, but now
' we can add to it new elements without causing an
' "object reference not set to an instance of an object" error.
_BodyHtml = New HtmlNodeCollection(FullHtml.DocumentNode)
FullHtml = (HtmlWeb.Load(Url))
' Fix any nodes error that may
' occur inside the html code.
FullHtml.OptionFixNestedTags = True
BodyHtml = FullHtml.DocumentNode.SelectNodes(BodyPath)
End Sub
End Class