2
\$\begingroup\$

I've been wanting to go async with my HTTP calls but all the methods I tried have not worked, so I resolved to implement this as a task and then improve upon it.

This is what I've come up with so far. Any noticeable improvements or unused techniques to make this go even faster and still produce viable images?

Public Shared Sub ProcessFolder(targetFolder As String)
    Dim ListParams As New List(Of ImgThreadParams)()
    Dim gtin As String = vbNullString
    Dim files = GetXlsxFiles(targetFolder)
    Dim excel As Application = New Application()
    Try

        For Each fileName As String In files
            'Dim book = New LinqToExcel.ExcelQueryFactory(System.IO.Path.GetFileName(fileName))
            Dim w As Workbook = excel.Workbooks.Open(fileName)
            ' Get sheet.
            Dim sheet As Worksheet = w.Sheets(1)
            ' Get range.
            Dim r As Range = sheet.UsedRange()
            'move range to 2d array
            Dim rArray(,) As Object = r.Value(XlRangeValueDataType.xlRangeValueDefault)
            'setup variables to hold column numbers for quick parsing in the next loop 
            Dim GtinC As Integer = 0
            Dim Img1 As Integer = 0
            Dim Img2 As Integer = 0
            Dim Img3 As Integer = 0
            Dim Img4 As Integer = 0
            Dim Img5 As Integer = 0
            Dim Img6 As Integer = 0
            Dim Img7 As Integer = 0
            Dim Img8 As Integer = 0
            Dim Img9 As Integer = 0
            Dim Img10 As Integer = 0
            Dim ic As Integer = 0
            'pull the desired data from the range without looping over undesired data.
            For j As Integer = 1 To r.Columns.Count()
                If rArray(4, j).ToString().StartsWith("GTIN") Then
                    GtinC = j
                ElseIf rArray(4, j).ToString.StartsWith("Image") And Not rArray(4, j).ToString().EndsWith("Description") And Not rArray(4, j).ToString().Contains("File") Then
                    ic = ic + 1
                    Select Case ic
                        Case 1
                            Img1 = j
                        Case 2
                            Img2 = j
                        Case 3
                            Img3 = j
                        Case 4
                            Img4 = j
                        Case 5
                            Img5 = j
                        Case 6
                            Img6 = j
                        Case 7
                            Img7 = j
                        Case 8
                            Img8 = j
                        Case 9
                            Img9 = j
                        Case 10
                            Img10 = j
                        Case Else
                            Exit Select
                    End Select
                End If
            Next
            Dim additem As Boolean = False
            'Do Some checking on the quality of the links passed..
            For i As Integer = 5 To r.Rows.Count()
                Dim itp As ImgThreadParams = New ImgThreadParams()
                itp.gtin = rArray(i, GtinC)
                If (rArray(i, Img1) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img1), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img1))
                        additem = True
                    End If
                End If
                If (rArray(i, Img2) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img2), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img2))
                    End If
                End If
                If (rArray(i, Img3) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img3), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img3))
                    End If
                End If
                If (rArray(i, Img4) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img4), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img4))
                    End If
                End If
                If (rArray(i, Img5) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img5), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img5))
                    End If
                End If
                If (rArray(i, Img6) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img6), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img6))
                    End If
                End If
                If (rArray(i, Img7) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img7), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img7))
                    End If
                End If
                If (rArray(i, Img8) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img8), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img8))
                    End If
                End If
                If (rArray(i, Img9) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img9), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img9))
                    End If
                End If
                If (rArray(i, Img10) IsNot Nothing) Then
                    If (Uri.IsWellFormedUriString(rArray(i, Img3), UriKind.Absolute)) Then
                        itp.lUrls.Add(rArray(i, Img10))
                    End If
                End If
                If (additem = True) Then
                    ListParams.Add(itp)
                    additem = False
                End If
            Next
            'Clean Up
            rArray = Nothing
            w.Close()
            Runtime.InteropServices.Marshal.ReleaseComObject(r)
            Runtime.InteropServices.Marshal.ReleaseComObject(sheet)
            Runtime.InteropServices.Marshal.ReleaseComObject(w)
            Runtime.InteropServices.Marshal.ReleaseComObject(excel)
        Next

        Dim Folder As DirectoryInfo = System.IO.Directory.CreateDirectory(System.IO.Directory.GetCurrentDirectory() + "\Image")
        If ListParams.Count > 4 Then
            Dim tCount As Integer = Math.Round(ListParams.Count / 10, MidpointRounding.ToEven)
            Dim L1 As New List(Of ImgThreadParams)
            For i As Integer = 0 To tCount
                L1.Add(ListParams(i))
            Next
            Dim L2 As New List(Of ImgThreadParams)
            For i As Integer = tCount + 1 To tCount * 2
                L2.Add(ListParams(i))
            Next
            Dim L3 As New List(Of ImgThreadParams)
            For i As Integer = tCount * 2 + 1 To tCount * 3
                L3.Add(ListParams(i))
            Next
            Dim L4 As New List(Of ImgThreadParams)
            For i As Integer = tCount * 3 + 1 To tCount * 4
                L4.Add(ListParams(i))
            Next
            Dim L5 As New List(Of ImgThreadParams)
            For i As Integer = tCount * 4 + 1 To tCount * 5
                L5.Add(ListParams(i))
            Next
            Dim L6 As New List(Of ImgThreadParams)
            For i As Integer = tCount * 5 + 1 To tCount * 6
                L6.Add(ListParams(i))
            Next
            Dim L7 As New List(Of ImgThreadParams)
            For i As Integer = tCount * 6 + 1 To tCount * 7
                L7.Add(ListParams(i))
            Next
            Dim L8 As New List(Of ImgThreadParams)
            For i As Integer = tCount * 7 + 1 To tCount * 8
                L8.Add(ListParams(i))
            Next
            Dim L9 As New List(Of ImgThreadParams)
            For i As Integer = tCount * 8 + 1 To tCount * 9
                L9.Add(ListParams(i))
            Next
            Dim L10 As New List(Of ImgThreadParams)
            For i As Integer = tCount * 9 + 1 To ListParams.Count - 1
                L10.Add(ListParams(i))
            Next
            Dim t1, t2, t3, t4, t5, t6, t7, t8, t9, t10 As Threading.Thread
            t1 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t2 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t3 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t4 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t5 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t6 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t7 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t8 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t9 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t10 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t1.Start(L1)
            t2.Start(L2)
            t3.Start(L3)
            t4.Start(L4)
            t5.Start(L5)
            t6.Start(L6)
            t7.Start(L7)
            t8.Start(L8)
            t9.Start(L9)
            t10.Start(L10)
        Else
            Dim t1 As Threading.Thread
            t1 = New Threading.Thread(AddressOf ThreadedGetGtinImages)
            t1.Start(ListParams)
        End If
    Catch ex As Exception

    End Try
End Sub
Public Shared pc As Integer = 1
Public Shared imgCount As Integer = 1
Public Shared Sub pCompleted()
    Console.WriteLine((pc * 10).ToString() + "% Completed")
    pc = pc + 1
End Sub
'I am not entirely sure this is the proper way to iterate over a lists in a list but it seems to work..

Public Shared Async Sub ThreadedGetGtinImages(ThreadParams As List(Of ImgThreadParams))
    Dim items As IEnumerable(Of Task(Of Boolean)) = From item As ImgThreadParams In ThreadParams
                                                    From url In item.lUrls
                                                    Where url IsNot Nothing
                                                    Select GetGtinImages(item.gtin, url)
    Dim downloadItems As Task(Of Boolean)() = items.ToArray()
    Dim bReturns As Boolean() = Await Task.WhenAll(downloadItems)
    pCompleted()
End Sub
Public Shared Function GetGtinImages(sGTIN As String, sURL As String, Optional sFolder As String = "") As Task(Of Boolean)

    Return Task.Factory.StartNew(Of Boolean)(
        Function() As Boolean
            Dim sFileExtension As String
            Using wc As WebClient = New WebClient()
                Dim myWebHeaderCollection As WebHeaderCollection

                Try
                    Dim dBytes As Byte() = wc.DownloadData(address:=sURL)
                    If (wc.ResponseHeaders(HttpResponseHeader.ContentLength) > 0) Then

                        myWebHeaderCollection = wc.ResponseHeaders

                        Dim sContentType As String = myWebHeaderCollection.Get("Content-Type")
                        If (sContentType.Length > 0) Then

                            Dim sImageContents As String() = sContentType.Split(";")

                            Dim sImage As String = sImageContents(0).Split("/")(0).ToLower

                            Dim sImageType As String = sImageContents(0).Split("/")(1).ToLower

                            If sImage.Equals("image") = True Then
                                Select Case sImageType
                                    Case "jpeg"
                                        sFileExtension = ".jpg"
                                    Case "png"
                                        sFileExtension = ".png"
                                    Case "bmp"
                                        sFileExtension = ".bmp"
                                    Case "gif"
                                        sFileExtension = ".gif"
                                    Case "tiff"
                                        sFileExtension = ".Tiff"
                                    Case Else
                                        Return False
                                End Select
                                sFolder = System.IO.Directory.GetCurrentDirectory() + "\Image"
                                Using Fi As FileStream = New FileStream(sFolder + "\" + sGTIN + "_" + imgCount.ToString + sFileExtension, FileMode.CreateNew)
                                    Fi.Write(dBytes, 0, dBytes.Length)
                                    Fi.Flush()
                                    Fi.Close()
                                End Using
                                imgCount = imgCount + 1
                                Return True
                            End If
                        End If
                    End If
                Catch ex As Exception
                    wc.Dispose()
                    Return False
                End Try
            End Using
            Return True
        End Function)
End Function
Public Shared Sub Main()
    ProcessFolder(System.IO.Directory.GetCurrentDirectory())
    Console.ReadLine()
End Sub
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.