4
\$\begingroup\$

I wrote this quick (lots of googling) and dirty (I hope not too dirty) script to download a URL list with a win32 wget port and save them as files with the url as the name to use as a "poor man's url cache" for another program.

It works (tested ~200k links), but I wonder if I did progress management right (it was a power shell feature I wanted to test with this kind of task) and if there are other things I could improve (or make more readable/reduce script length) and/or unexpected failure points.

It is my first time using powershell like this ... please be gentle ;P

param ([Parameter(Mandatory = $true)][string]$target = 'default'); cd $PSScriptRoot;

$enableProgress = $true;

#if ($enableProgress){

Add-Type -AssemblyName System.Web

$urlPAth = "$( pwd )\url_$target.txt";

$baseCrawlPath = "$( pwd )\crawl_data\$target";

if (![System.IO.File]::Exists($urlPath))
{
    Write-Error "missing list file"; exit -1
}

$foo = (mkdir $baseCrawlPath 2> $null);

Write-Progress -Id 13431 -Activity "Load database..." -PercentComplete -1 -Status "Reading file: $urlPath"
$totalLines = (gc ".\url_$target.txt" | ? { $_.Trim() -ne '' }).Count;
Write-Progress -Id 13431 -Activity "Load database..." -Completed

$totalCounter = 0;
$retMax = 10;
$interGetSeconds = 3;

$skipExisting = $true;

[IO.File]::ReadLines($urlPath) | Where-Object { $_.Trim() -ne '' } | ForEach-Object {
    $totalCounter++;

    #$host.ui.RawUI.WindowTitle = "$totalLines || $totalCounter"
    # i scrapped this because wget percentage hijacks the title

    if ($enableProgress)
    {
        Write-Progress -Id 1 -Activity "DOWNLOAD - ($target)" -Status "Item $totalCounter of $totalLines" -PercentComplete (($totalCounter / $totalLines) * 100);
    }

    $theUrl = $_.Trim(); $retCount = 0;
    $fileName = $theUrl;
    # I url encode invalid path characters so that the url is preserved when url-decoding all files in folder
    [IO.Path]::GetInvalidFileNameChars() | foreach { $fileName = $fileName.Replace([String]$_,[System.Web.HttpUtility]::UrlEncode($_)) }
    $filePath = "$baseCrawlPath\$fileName"; # echo $filePath; # debug

    $keepTrying = ($retCount -lt $retMax);
    # $keepTrying = $true; # debug

    $fExists = $false;
    if ($skipExisting)
    {
        $fExists = [System.IO.File]::Exists($filePath);
    }
    if ($fExists)
    {
        $errored = $false; $keepTrying = $false
    }

    while ($keepTrying)
    {
        $retCount++; 
        $keepTrying = ($retCount -lt $retMax);

        $rtPct = (($retCount/$retMax)*100);

        if ($rtPct -gt 100)
        {
            $rtPct = -1
        }
        $rtPct = -1

        if ($enableProgress)
        {
            Write-Progress -Id 2 -ParentId 1 -Activity "WGET [$_]" -Status "Attempt $retCount of $retMax" -PercentComplete $rtPct
        }

        $wgOut = '';

        # $stderrFile = New-TemporaryFile; $stdoutOutput = cmd /c ver '&' dir \nosuch 2>$stderrFile; $stderrOutput = Get-Content $stdErrFile; Remove-Item $stderrFile

        # wget.exe -N "$_"; // timestamping was my initial idea but apparently I can't use urls as filenames

        wget.exe "$_" -O "$filePath"; 
        $wreq = $true; # enable wait

        # i failed at putting stdout and stderr in a variable...
        #Invoke-Expression "C:\IncludePath\wget.exe \"$_\" -O \"$filePath\"" -OutVariable wgOut -ErrorVariable wgOut
        #echo (-join('C:\IncludePath\wget.exe',' "',$_,'"',' -O "',$filePath,'"'));
        #$stderrFile = New-TemporaryFile; $stdoutOutput = Invoke-Expression (-join('C:\IncludePath\wget.exe',' "',$_,'"',' -O "',$filePath,'"')) 2>$stderrFile; $stderrOutput = Get-Content $stdErrFile; Remove-Item $stderrFile

        $errored = ($LASTEXITCODE -ne 0); #  if($errored){pause} # debug

        if ($errored)
        {
            #echo $stderrOutput; // initial idea was to have streams in variables to print only in case of errors.
            #echo $stdoutOutput;

            For ($iTimeCounter1 = ($interGetSeconds*$retCount + 3)*5; $iTimeCounter1 -gt 0; $iTimeCounter1--) {
                if ($enableProgress)
                {
                    Write-Progress -Id 4 -ParentId 2 -Activity "ZEN MEDITATION ..." -Status "WGET ERROR CODE: $LASTEXITCODE" -SecondsRemaining $iTimeCounter1;
                }
                Start-Sleep 1;
            }
            if ($enableProgress)
            {
                Write-Progress -Id 4 -ParentId 2 -Activity "ZEN MEDITATION ..." -Completed
            }
            #pause # debug
        }
        else
        {
            if ($enableProgress)
            {
                Write-Progress -Id 2 -ParentId 1 -Activity "WGET [$_]" -Status "Done. Attempt n. $retCount." -PercentComplete -1
            }
            break;
        }

    } # END WHILE


    if ($wreq) # delay requests
    {
        For ($iTimeCounter1 = $interGetSeconds; $iTimeCounter1 -gt 0; $iTimeCounter1--) {
            if ($enableProgress)
            {
                Write-Progress -Id 2 -ParentId 1 -Activity "WGET [$_]" -Status "Done. Attempt n. $retCount." -CurrentOperation "Attesa tra una richiesta e l'altra..." -SecondsRemaining $iTimeCounter1;
            }
            Start-Sleep 1;
        }
    };


    if ($enableProgress)
    {
        Write-Progress -Id 2 -ParentId 1 -Activity "WGET [$_]" -Status "Done. Attempt n. $retCount." -Completed
    }

    if ($errored)
    {
        pause
    } # debug

}
\$\endgroup\$
1
  • 1
    \$\begingroup\$ You can even download files without using wget even showing the progressbar Invoke-WebRequest Url -OutFile OutputFile \$\endgroup\$
    – Wasif
    Feb 11, 2020 at 8:17

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.