I've been wanting to go async with my HTTP calls but all the methods I tried have not worked, so I resolved to implement this as a task and then improve upon it.
This is what I've come up with so far. Any noticeable improvements or unused techniques to make this go even faster and still produce viable images?
Public Shared Sub ProcessFolder(targetFolder As String)
Dim ListParams As New List(Of ImgThreadParams)()
Dim gtin As String = vbNullString
Dim files = GetXlsxFiles(targetFolder)
Dim excel As Application = New Application()
Try
For Each fileName As String In files
'Dim book = New LinqToExcel.ExcelQueryFactory(System.IO.Path.GetFileName(fileName))
Dim w As Workbook = excel.Workbooks.Open(fileName)
' Get sheet.
Dim sheet As Worksheet = w.Sheets(1)
' Get range.
Dim r As Range = sheet.UsedRange()
'move range to 2d array
Dim rArray(,) As Object = r.Value(XlRangeValueDataType.xlRangeValueDefault)
'setup variables to hold column numbers for quick parsing in the next loop
Dim GtinC As Integer = 0
Dim Img1 As Integer = 0
Dim Img2 As Integer = 0
Dim Img3 As Integer = 0
Dim Img4 As Integer = 0
Dim Img5 As Integer = 0
Dim Img6 As Integer = 0
Dim Img7 As Integer = 0
Dim Img8 As Integer = 0
Dim Img9 As Integer = 0
Dim Img10 As Integer = 0
Dim ic As Integer = 0
'pull the desired data from the range without looping over undesired data.
For j As Integer = 1 To r.Columns.Count()
If rArray(4, j).ToString().StartsWith("GTIN") Then
GtinC = j
ElseIf rArray(4, j).ToString.StartsWith("Image") And Not rArray(4, j).ToString().EndsWith("Description") And Not rArray(4, j).ToString().Contains("File") Then
ic = ic + 1
Select Case ic
Case 1
Img1 = j
Case 2
Img2 = j
Case 3
Img3 = j
Case 4
Img4 = j
Case 5
Img5 = j
Case 6
Img6 = j
Case 7
Img7 = j
Case 8
Img8 = j
Case 9
Img9 = j
Case 10
Img10 = j
Case Else
Exit Select
End Select
End If
Next
Dim additem As Boolean = False
'Do Some checking on the quality of the links passed..
For i As Integer = 5 To r.Rows.Count()
Dim itp As ImgThreadParams = New ImgThreadParams()
itp.gtin = rArray(i, GtinC)
If (rArray(i, Img1) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img1), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img1))
additem = True
End If
End If
If (rArray(i, Img2) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img2), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img2))
End If
End If
If (rArray(i, Img3) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img3), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img3))
End If
End If
If (rArray(i, Img4) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img4), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img4))
End If
End If
If (rArray(i, Img5) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img5), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img5))
End If
End If
If (rArray(i, Img6) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img6), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img6))
End If
End If
If (rArray(i, Img7) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img7), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img7))
End If
End If
If (rArray(i, Img8) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img8), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img8))
End If
End If
If (rArray(i, Img9) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img9), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img9))
End If
End If
If (rArray(i, Img10) IsNot Nothing) Then
If (Uri.IsWellFormedUriString(rArray(i, Img3), UriKind.Absolute)) Then
itp.lUrls.Add(rArray(i, Img10))
End If
End If
If (additem = True) Then
ListParams.Add(itp)
additem = False
End If
Next
'Clean Up
rArray = Nothing
w.Close()
Runtime.InteropServices.Marshal.ReleaseComObject(r)
Runtime.InteropServices.Marshal.ReleaseComObject(sheet)
Runtime.InteropServices.Marshal.ReleaseComObject(w)
Runtime.InteropServices.Marshal.ReleaseComObject(excel)
Next
Dim Folder As DirectoryInfo = System.IO.Directory.CreateDirectory(System.IO.Directory.GetCurrentDirectory() + "\Image")
If ListParams.Count > 4 Then
Dim tCount As Integer = Math.Round(ListParams.Count / 10, MidpointRounding.ToEven)
Dim L1 As New List(Of ImgThreadParams)
For i As Integer = 0 To tCount
L1.Add(ListParams(i))
Next
Dim L2 As New List(Of ImgThreadParams)
For i As Integer = tCount + 1 To tCount * 2
L2.Add(ListParams(i))
Next
Dim L3 As New List(Of ImgThreadParams)
For i As Integer = tCount * 2 + 1 To tCount * 3
L3.Add(ListParams(i))
Next
Dim L4 As New List(Of ImgThreadParams)
For i As Integer = tCount * 3 + 1 To tCount * 4
L4.Add(ListParams(i))
Next
Dim L5 As New List(Of ImgThreadParams)
For i As Integer = tCount * 4 + 1 To tCount * 5
L5.Add(ListParams(i))
Next
Dim L6 As New List(Of ImgThreadParams)
For i As Integer = tCount * 5 + 1 To tCount * 6
L6.Add(ListParams(i))
Next
Dim L7 As New List(Of ImgThreadParams)
For i As Integer = tCount * 6 + 1 To tCount * 7
L7.Add(ListParams(i))
Next
Dim L8 As New List(Of ImgThreadParams)
For i As Integer = tCount * 7 + 1 To tCount * 8
L8.Add(ListParams(i))
Next
Dim L9 As New List(Of ImgThreadParams)
For i As Integer = tCount * 8 + 1 To tCount * 9
L9.Add(ListParams(i))
Next
Dim L10 As New List(Of ImgThreadParams)
For i As Integer = tCount * 9 + 1 To ListParams.Count - 1
L10.Add(ListParams(i))
Next
Dim t1, t2, t3, t4, t5, t6, t7, t8, t9, t10 As Threading.Thread
t1 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t2 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t3 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t4 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t5 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t6 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t7 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t8 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t9 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t10 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t1.Start(L1)
t2.Start(L2)
t3.Start(L3)
t4.Start(L4)
t5.Start(L5)
t6.Start(L6)
t7.Start(L7)
t8.Start(L8)
t9.Start(L9)
t10.Start(L10)
Else
Dim t1 As Threading.Thread
t1 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
t1.Start(ListParams)
End If
Catch ex As Exception
End Try
End Sub
Public Shared pc As Integer = 1
Public Shared imgCount As Integer = 1
Public Shared Sub pCompleted()
Console.WriteLine((pc * 10).ToString() + "% Completed")
pc = pc + 1
End Sub
'I am not entirely sure this is the proper way to iterate over a lists in a list but it seems to work..
Public Shared Async Sub ThreadedGetGtinImages(ThreadParams As List(Of ImgThreadParams))
Dim items As IEnumerable(Of Task(Of Boolean)) = From item As ImgThreadParams In ThreadParams
From url In item.lUrls
Where url IsNot Nothing
Select GetGtinImages(item.gtin, url)
Dim downloadItems As Task(Of Boolean)() = items.ToArray()
Dim bReturns As Boolean() = Await Task.WhenAll(downloadItems)
pCompleted()
End Sub
Public Shared Function GetGtinImages(sGTIN As String, sURL As String, Optional sFolder As String = "") As Task(Of Boolean)
Return Task.Factory.StartNew(Of Boolean)(
Function() As Boolean
Dim sFileExtension As String
Using wc As WebClient = New WebClient()
Dim myWebHeaderCollection As WebHeaderCollection
Try
Dim dBytes As Byte() = wc.DownloadData(address:=sURL)
If (wc.ResponseHeaders(HttpResponseHeader.ContentLength) > 0) Then
myWebHeaderCollection = wc.ResponseHeaders
Dim sContentType As String = myWebHeaderCollection.Get("Content-Type")
If (sContentType.Length > 0) Then
Dim sImageContents As String() = sContentType.Split(";")
Dim sImage As String = sImageContents(0).Split("/")(0).ToLower
Dim sImageType As String = sImageContents(0).Split("/")(1).ToLower
If sImage.Equals("image") = True Then
Select Case sImageType
Case "jpeg"
sFileExtension = ".jpg"
Case "png"
sFileExtension = ".png"
Case "bmp"
sFileExtension = ".bmp"
Case "gif"
sFileExtension = ".gif"
Case "tiff"
sFileExtension = ".Tiff"
Case Else
Return False
End Select
sFolder = System.IO.Directory.GetCurrentDirectory() + "\Image"
Using Fi As FileStream = New FileStream(sFolder + "\" + sGTIN + "_" + imgCount.ToString + sFileExtension, FileMode.CreateNew)
Fi.Write(dBytes, 0, dBytes.Length)
Fi.Flush()
Fi.Close()
End Using
imgCount = imgCount + 1
Return True
End If
End If
End If
Catch ex As Exception
wc.Dispose()
Return False
End Try
End Using
Return True
End Function)
End Function
Public Shared Sub Main()
ProcessFolder(System.IO.Directory.GetCurrentDirectory())
Console.ReadLine()
End Sub
