I wrote this quick (lots of googling) and dirty (I hope not too dirty) script to download a URL list with a win32 wget port and save them as files with the url as the name to use as a "poor man's url cache" for another program.
It works (tested ~200k links), but I wonder if I did progress management right (it was a power shell feature I wanted to test with this kind of task) and if there are other things I could improve (or make more readable/reduce script length) and/or unexpected failure points.
It is my first time using powershell like this ... please be gentle ;P
param ([Parameter(Mandatory = $true)][string]$target = 'default'); cd $PSScriptRoot;
$enableProgress = $true;
#if ($enableProgress){
Add-Type -AssemblyName System.Web
$urlPAth = "$( pwd )\url_$target.txt";
$baseCrawlPath = "$( pwd )\crawl_data\$target";
if (![System.IO.File]::Exists($urlPath))
{
Write-Error "missing list file"; exit -1
}
$foo = (mkdir $baseCrawlPath 2> $null);
Write-Progress -Id 13431 -Activity "Load database..." -PercentComplete -1 -Status "Reading file: $urlPath"
$totalLines = (gc ".\url_$target.txt" | ? { $_.Trim() -ne '' }).Count;
Write-Progress -Id 13431 -Activity "Load database..." -Completed
$totalCounter = 0;
$retMax = 10;
$interGetSeconds = 3;
$skipExisting = $true;
[IO.File]::ReadLines($urlPath) | Where-Object { $_.Trim() -ne '' } | ForEach-Object {
$totalCounter++;
#$host.ui.RawUI.WindowTitle = "$totalLines || $totalCounter"
# i scrapped this because wget percentage hijacks the title
if ($enableProgress)
{
Write-Progress -Id 1 -Activity "DOWNLOAD - ($target)" -Status "Item $totalCounter of $totalLines" -PercentComplete (($totalCounter / $totalLines) * 100);
}
$theUrl = $_.Trim(); $retCount = 0;
$fileName = $theUrl;
# I url encode invalid path characters so that the url is preserved when url-decoding all files in folder
[IO.Path]::GetInvalidFileNameChars() | foreach { $fileName = $fileName.Replace([String]$_,[System.Web.HttpUtility]::UrlEncode($_)) }
$filePath = "$baseCrawlPath\$fileName"; # echo $filePath; # debug
$keepTrying = ($retCount -lt $retMax);
# $keepTrying = $true; # debug
$fExists = $false;
if ($skipExisting)
{
$fExists = [System.IO.File]::Exists($filePath);
}
if ($fExists)
{
$errored = $false; $keepTrying = $false
}
while ($keepTrying)
{
$retCount++;
$keepTrying = ($retCount -lt $retMax);
$rtPct = (($retCount/$retMax)*100);
if ($rtPct -gt 100)
{
$rtPct = -1
}
$rtPct = -1
if ($enableProgress)
{
Write-Progress -Id 2 -ParentId 1 -Activity "WGET [$_]" -Status "Attempt $retCount of $retMax" -PercentComplete $rtPct
}
$wgOut = '';
# $stderrFile = New-TemporaryFile; $stdoutOutput = cmd /c ver '&' dir \nosuch 2>$stderrFile; $stderrOutput = Get-Content $stdErrFile; Remove-Item $stderrFile
# wget.exe -N "$_"; // timestamping was my initial idea but apparently I can't use urls as filenames
wget.exe "$_" -O "$filePath";
$wreq = $true; # enable wait
# i failed at putting stdout and stderr in a variable...
#Invoke-Expression "C:\IncludePath\wget.exe \"$_\" -O \"$filePath\"" -OutVariable wgOut -ErrorVariable wgOut
#echo (-join('C:\IncludePath\wget.exe',' "',$_,'"',' -O "',$filePath,'"'));
#$stderrFile = New-TemporaryFile; $stdoutOutput = Invoke-Expression (-join('C:\IncludePath\wget.exe',' "',$_,'"',' -O "',$filePath,'"')) 2>$stderrFile; $stderrOutput = Get-Content $stdErrFile; Remove-Item $stderrFile
$errored = ($LASTEXITCODE -ne 0); # if($errored){pause} # debug
if ($errored)
{
#echo $stderrOutput; // initial idea was to have streams in variables to print only in case of errors.
#echo $stdoutOutput;
For ($iTimeCounter1 = ($interGetSeconds*$retCount + 3)*5; $iTimeCounter1 -gt 0; $iTimeCounter1--) {
if ($enableProgress)
{
Write-Progress -Id 4 -ParentId 2 -Activity "ZEN MEDITATION ..." -Status "WGET ERROR CODE: $LASTEXITCODE" -SecondsRemaining $iTimeCounter1;
}
Start-Sleep 1;
}
if ($enableProgress)
{
Write-Progress -Id 4 -ParentId 2 -Activity "ZEN MEDITATION ..." -Completed
}
#pause # debug
}
else
{
if ($enableProgress)
{
Write-Progress -Id 2 -ParentId 1 -Activity "WGET [$_]" -Status "Done. Attempt n. $retCount." -PercentComplete -1
}
break;
}
} # END WHILE
if ($wreq) # delay requests
{
For ($iTimeCounter1 = $interGetSeconds; $iTimeCounter1 -gt 0; $iTimeCounter1--) {
if ($enableProgress)
{
Write-Progress -Id 2 -ParentId 1 -Activity "WGET [$_]" -Status "Done. Attempt n. $retCount." -CurrentOperation "Attesa tra una richiesta e l'altra..." -SecondsRemaining $iTimeCounter1;
}
Start-Sleep 1;
}
};
if ($enableProgress)
{
Write-Progress -Id 2 -ParentId 1 -Activity "WGET [$_]" -Status "Done. Attempt n. $retCount." -Completed
}
if ($errored)
{
pause
} # debug
}
Invoke-WebRequest Url -OutFile OutputFile
\$\endgroup\$