From 56b9b2c6567003130ca783aae577c5d191c46cf2 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Fri, 26 Apr 2024 07:22:46 +0200 Subject: [PATCH] fix download timeout (#3514) Downloading artifacts would hang forever some times. Co-authored-by: freddydk --- Artifacts/Download-Artifacts.ps1 | 2 +- BC.HelperFunctions.ps1 | 2 + Common/Download-File.ps1 | 3 +- HelperFunctions.ps1 | 144 +++++++------------------------ 4 files changed, 36 insertions(+), 115 deletions(-) diff --git a/Artifacts/Download-Artifacts.ps1 b/Artifacts/Download-Artifacts.ps1 index ed5e3d1db..a8258fefa 100644 --- a/Artifacts/Download-Artifacts.ps1 +++ b/Artifacts/Download-Artifacts.ps1 @@ -30,7 +30,7 @@ function Download-Artifacts { [switch] $force, [switch] $forceRedirection, [string] $basePath = "", - [int] $timeout = 300 + [int] $timeout = $bccontainerHelperConfig.artifactDownloadTimeout ) function DownloadPackage { diff --git a/BC.HelperFunctions.ps1 b/BC.HelperFunctions.ps1 index 5e7504d42..b97734ef1 100644 --- a/BC.HelperFunctions.ps1 +++ b/BC.HelperFunctions.ps1 @@ -35,6 +35,8 @@ function Get-ContainerHelperConfig { "useSharedEncryptionKeys" = $true "DOCKER_SCAN_SUGGEST" = $false "psSessionTimeout" = 0 + "artifactDownloadTimeout" = 300 + "defaultDownloadTimeout" = 100 "baseUrl" = "https://businesscentral.dynamics.com" "apiBaseUrl" = "https://api.businesscentral.dynamics.com" "mapCountryCode" = [PSCustomObject]@{ diff --git a/Common/Download-File.ps1 b/Common/Download-File.ps1 index e9e0c33a3..fc4ca6022 100644 --- a/Common/Download-File.ps1 +++ b/Common/Download-File.ps1 @@ -27,7 +27,7 @@ function Download-File { [string] $description = '', [hashtable] $headers = @{"UserAgent" = "BcContainerHelper $bcContainerHelperVersion" }, [switch] $dontOverwrite, - [int] $timeout = 100 + [int] $timeout = $bccontainerHelperConfig.defaultDownloadTimeout ) $replaceUrls = @{ @@ -86,6 +86,7 @@ function Download-File { } else { Write-Host "Could not download from CDN..., retrying from blob storage in $waittime seconds..." + $timeout += $timeout } Start-Sleep -Seconds $waittime DownloadFileLow -sourceUrl $newSourceUrl -destinationFile $destinationFile -dontOverwrite:$dontOverwrite -timeout $timeout -headers $headers diff --git a/HelperFunctions.ps1 b/HelperFunctions.ps1 index 1c75250ee..02542d860 100644 --- a/HelperFunctions.ps1 +++ b/HelperFunctions.ps1 @@ -1,35 +1,4 @@ -$useTimeOutWebClient = $false -if ($PSVersionTable.PSVersion -lt "6.0.0" -or $useTimeOutWebClient) { - $timeoutWebClientCode = @" - using System.Net; - - public class TimeoutWebClient : WebClient - { - int theTimeout; - - public TimeoutWebClient(int timeout) - { - theTimeout = timeout; - } - - protected override WebRequest GetWebRequest(System.Uri address) - { - WebRequest request = base.GetWebRequest(address); - if (request != null) - { - request.Timeout = theTimeout; - } - return request; - } - } -"@; -if (-not ([System.Management.Automation.PSTypeName]"TimeoutWebClient").Type) { - Add-Type -TypeDefinition $timeoutWebClientCode -Language CSharp -WarningAction SilentlyContinue | Out-Null - $useTimeOutWebClient = $true -} -} - -$sslCallbackCode = @" +$sslCallbackCode = @" using System.Net.Security; using System.Security.Cryptography.X509Certificates; @@ -967,27 +936,6 @@ function GetHash { (Get-FileHash -InputStream $stream -Algorithm SHA256).Hash } -function Wait-Task { - param( - [Parameter(Mandatory, ValueFromPipeline)] - [System.Threading.Tasks.Task[]]$Task - ) - - Begin { - $Tasks = @() - } - - Process { - $Tasks += $Task - } - - End { - While (-not [System.Threading.Tasks.Task]::WaitAll($Tasks, 200)) {} - $Tasks.ForEach( { $_.GetAwaiter().GetResult() }) - } -} -Set-Alias -Name await -Value Wait-Task -Force - function DownloadFileLow { Param( [string] $sourceUrl, @@ -999,71 +947,41 @@ function DownloadFileLow { [int] $timeout = 100 ) - if ($useTimeOutWebClient) { - Write-Host "Downloading using WebClient" - if ($skipCertificateCheck) { - Write-Host "Disabling SSL Verification" - [SslVerification]::Disable() - } - $webClient = New-Object TimeoutWebClient -ArgumentList (1000 * $timeout) - $headers.Keys | ForEach-Object { - $webClient.Headers.Add($_, $headers."$_") - } - $webClient.UseDefaultCredentials = $useDefaultCredentials - if (Test-Path $destinationFile -PathType Leaf) { - if ($dontOverwrite) { - return - } - Remove-Item -Path $destinationFile -Force - } - try { - $webClient.DownloadFile($sourceUrl, $destinationFile) - } - finally { - $webClient.Dispose() - if ($skipCertificateCheck) { - Write-Host "Restoring SSL Verification" - [SslVerification]::Enable() - } - } + $handler = New-Object System.Net.Http.HttpClientHandler + if ($skipCertificateCheck) { + Write-Host "Disabling SSL Verification on HttpClient" + [SslVerification]::DisableSsl($handler) + } + if ($useDefaultCredentials) { + $handler.UseDefaultCredentials = $true + } + $httpClient = New-Object System.Net.Http.HttpClient -ArgumentList $handler + $httpClient.Timeout = [Timespan]::FromSeconds($timeout) + $headers.Keys | ForEach-Object { + $httpClient.DefaultRequestHeaders.Add($_, $headers."$_") + } + $stream = $null + $fileStream = $null + if ($dontOverwrite) { + $fileMode = [System.IO.FileMode]::CreateNew } else { - Write-Host "Downloading using HttpClient" - - $handler = New-Object System.Net.Http.HttpClientHandler - if ($skipCertificateCheck) { - Write-Host "Disabling SSL Verification on HttpClient" - [SslVerification]::DisableSsl($handler) - } - if ($useDefaultCredentials) { - $handler.UseDefaultCredentials = $true - } - $httpClient = New-Object System.Net.Http.HttpClient -ArgumentList $handler - $httpClient.Timeout = [Timespan]::FromSeconds($timeout) - $headers.Keys | ForEach-Object { - $httpClient.DefaultRequestHeaders.Add($_, $headers."$_") - } - $stream = $null - $fileStream = $null - if ($dontOverwrite) { - $fileMode = [System.IO.FileMode]::CreateNew - } - else { - $fileMode = [System.IO.FileMode]::Create + $fileMode = [System.IO.FileMode]::Create + } + try { + $stream = $httpClient.GetStreamAsync($sourceUrl).GetAwaiter().GetResult() + $fileStream = New-Object System.IO.Filestream($destinationFile, $fileMode) + if (-not $stream.CopyToAsync($fileStream).Wait($timeout * 1000)) { + throw "Timeout downloading file" } - try { - $stream = $httpClient.GetStreamAsync($sourceUrl).GetAwaiter().GetResult() - $fileStream = New-Object System.IO.Filestream($destinationFile, $fileMode) - $stream.CopyToAsync($fileStream).GetAwaiter().GetResult() | Out-Null + } + finally { + if ($fileStream) { $fileStream.Close() + $fileStream.Dispose() } - finally { - if ($fileStream) { - $fileStream.Dispose() - } - if ($stream) { - $stream.Dispose() - } + if ($stream) { + $stream.Dispose() } } }