diff --git a/doc/download.xml b/doc/download.xml index aa4d7ad..a4a8980 100644 --- a/doc/download.xml +++ b/doc/download.xml @@ -51,6 +51,17 @@ An optional record opt can be given. The following components are supported.

+maxTime + + If this component is bound then its value must be a nonnegative integer + n, meaning that the function gives up after n seconds. +

+ A zero value of n means that no timeout is set, the method will + never give up in this case. +

+ The default for n is given by the value of the user preference + DownloadMaxTime (see ). + target If this component is bound then its value must be a string @@ -63,7 +74,7 @@ The following components are supported. If this component is bound and has the value false then those download methods that are based on curl or wget will omit the check of the server's certificate. - +

The same effect is achieved for all calls by setting the user preference DownloadVerifyCertificate (see ) to false @@ -116,6 +127,24 @@ not an option, then disabling certificate checks may be a good last resort. + +User preference DownloadMaxTime +DownloadMaxTime + +The value 0 (the default) means that no timeout is set +in calls of . +If the value is a positive integer n then those download methods that +support a timeout will give up after n seconds. +

+One can set the value of the preference to be val via +, by calling +SetUserPreference( "utils", "DownloadMaxTime", val ), +and access the current value via +, by calling +UserPreference( "utils", "DownloadMaxTime" ). + + + diff --git a/lib/download.gd b/lib/download.gd index d669f91..78ae3ce 100644 --- a/lib/download.gd +++ b/lib/download.gd @@ -34,3 +34,21 @@ curl or wget will omit the check of the server's certificate." package:= "utils", ) ); + +############################################################################# +## +#U DownloadMaxTime +## +DeclareUserPreference( rec( + name:= "DownloadMaxTime", + description:= [ + "The value '0' (the default) means that no timeout is set \ +in calls of 'Download'. \ +If the value is a positive integer 'n' then those download methods that \ +support a timeout will give up after 'n' seconds." + ], + default:= 0, + check:= val -> val = 0 or IsPosInt( val ), + package:= "utils", + ) ); + diff --git a/lib/download.gi b/lib/download.gi index 9281a0e..21ef3a1 100644 --- a/lib/download.gi +++ b/lib/download.gi @@ -40,6 +40,7 @@ Add( Download_Methods, rec( if not IsBound( opt.failOnError ) then opt.failOnError:= true; fi; + # 'DownloadURL' handles the options 'verifyCert' and 'maxTime'. res:= ValueGlobal( "DownloadURL" )( url, opt ); if res.success = true and @@ -58,6 +59,8 @@ Add( Download_Methods, rec( if not StartsWith( url, "http://" ) then return rec( success:= false, error:= "protocol is not http" ); + elif IsBound( opt.maxTime ) and opt.maxTime <> 0 then + return rec( success:= false, error:= "no support for given timeout" ); fi; rurl:= ReplacedString( url, "http://", "" ); @@ -76,7 +79,8 @@ Add( Download_Methods, rec( error:= res.status ); elif res.statuscode >= 400 then return rec( success:= false, - error:= Concatenation( "HTTP error code ", res.statuscode ) ); + error:= Concatenation( "HTTP error code ", + String( res.statuscode ) ) ); elif not ( IsBound( opt.target ) and IsString( opt.target ) ) then return rec( success:= true, result:= res.body ); else @@ -95,6 +99,12 @@ Add( Download_Methods, rec( download:= function( url, opt ) local res, outstream, exec, args, code; + if IsBound( opt.maxTime ) and opt.maxTime <> 0 then + # wget 1.20.3 ignores a given timeout. + # (wget 1.21.3 would support timeout.) + return rec( success:= false, error:= "no support for given timeout" ); + fi; + res:= ""; outstream:= OutputTextString( res, true ); exec:= Filename( DirectoriesSystemPrograms(), "wget" ); @@ -106,6 +116,9 @@ Add( Download_Methods, rec( if IsBound( opt.verifyCert ) and opt.verifyCert = false then Add( args, "--no-check-certificate" ); fi; + if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then + Add( args, Concatenation( "--timeout=", String( opt.maxTime ) ) ); + fi; code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args ); CloseStream( outstream ); if code <> 0 then @@ -147,6 +160,10 @@ Add( Download_Methods, rec( else Add( args, "-" ); fi; + if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then + Add( args, "--max-time" ); + Add( args, opt.maxTime ); + fi; Add( args, url ); code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args ); CloseStream( outstream ); @@ -177,7 +194,7 @@ InstallMethod( Download, InstallMethod( Download, [ "IsString", "IsRecord" ], function( url, opt ) - local errors, r, res; + local timeout, errors, r, res; # Set the default for 'verifyCert' if necessary. if not IsBound( opt.verifyCert ) and @@ -185,6 +202,14 @@ InstallMethod( Download, opt.verifyCert:= false; fi; + # Set the default for 'maxTime' if necessary. + if not IsBound( opt.maxTime ) then + timeout:= UserPreference( "utils", "DownloadMaxTime" ); + if IsPosInt( timeout ) then + opt.maxTime:= timeout; + fi; + fi; + # Run over the methods. errors:= []; for r in Download_Methods do @@ -213,6 +238,11 @@ InstallMethod( Download, return rec( success:= false, error:= "no method was available" ); else # At least one method was tried but without success. + if IsBound( opt.maxTime ) then + Add( errors, + Concatenation( "(maxTime option was set to ", + String( opt.maxTime ), ")" ) ); + fi; return rec( success:= false, error:= JoinStringsWithSeparator( errors, "; " ) ); fi; diff --git a/tst/download.tst b/tst/download.tst index f07eea9..4718e44 100644 --- a/tst/download.tst +++ b/tst/download.tst @@ -1,4 +1,4 @@ -#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r; +#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad ############################################################################ ## #W download.tst Utils Package Thomas Breuer @@ -54,8 +54,10 @@ gap> for pair in urls do > if expected = false and Length( good2 ) > 0 then > Print( "success for url ", url, "?\n" ); > fi; -> if Length( good1 ) <> Length( good2 ) then -> Print( "different success cases for url ", url, "\n" ); +> if List( good1, x -> x[2] ) <> List( good2, x -> x[2] ) then +> Print( "different success cases for url ", url, ":\n", +> List( good1, x -> x[2] ), " vs. ", List( good2, x -> x[2] ), +> "\n" ); > fi; > if Length( good1 ) > 0 then > contents:= good1[1][1].result; @@ -65,8 +67,36 @@ gap> for pair in urls do > fi; > od; > fi; +> res3:= List( meths, +> r -> [ r.download( url, +> rec( maxTime:= 10 ) ), +> r.position ] );; +> good3:= Filtered( res3, r -> r[1].success = true );; +> if expected = false and Length( good3 ) > 0 then +> Print( "success for url ", url, "?\n" ); +> fi; +> # The IO and wget based methods are available. +> # They cannot handle the 'maxTime' parameter. +> bad:= Filtered( meths, +> x -> StartsWith( x.name, "via SingleHTTPRequest" ) or +> StartsWith( x.name, "via wget" ) ); +> bad:= List( bad, x -> x.position ); +> good1:= Filtered( good1, x -> not x[2] in bad ); +> if List( good1, x -> x[2] ) <> List( good3, x -> x[2] ) then +> Print( "different success cases for url ", url, ":\n", +> List( good1, x -> x[2] ), " vs. ", List( good3, x -> x[2] ), +> "\n" ); +> fi; > od; +## test timeout +gap> res1:= Download( "https://httpbun.com/delay/3", rec( maxTime:= 1 ) );; +gap> res1.success = false; +true +gap> res1:= Download( "https://httpbun.com/delay/3", rec( maxTime:= 5 ) );; +gap> res1.success = true; +true + ## the example 9.1.1 from the manual gap> url:= "https://www.gap-system.org/index.html";; gap> res1:= Download( url );;