support timeout in Download

doc/download.xml

@@ -51,6 +51,17 @@ An optional record <A>opt</A> can be given.
 The following components are supported.
 <P/>
 <List>
+<Mark><C>maxTime</C></Mark>
+<Item>
+  If this component is bound then its value must be a nonnegative integer
+  <M>n</M>, meaning that the function gives up after <M>n</M> seconds.
+  <P/>
+  A zero value of <M>n</M> means that no timeout is set, the method will
+  never give up in this case.
+  <P/>
+  The default for <M>n</M> is given by the value of the user preference
+  <C>DownloadMaxTime</C> (see <Ref Subsect="subsec-DownloadMaxTime"/>).
+</Item>
 <Mark><C>target</C></Mark>
 <Item>
   If this component is bound then its value must be a string
@@ -63,7 +74,7 @@ The following components are supported.
   If this component is bound and has the value <K>false</K>
   then those download methods that are based on <C>curl</C> or <C>wget</C>
   will omit the check of the server's certificate.
-
+  <P/>
   The same effect is achieved for all <Ref Func="Download"/> calls
   by setting the user preference <C>DownloadVerifyCertificate</C>
   (see <Ref Subsect="subsec-DownloadVerifyCertificate"/>) to <K>false</K>
@@ -116,6 +127,24 @@ not an option, then disabling certificate checks may be a good last resort.
 
 </Subsection>
 
+<Subsection Label="subsec-DownloadMaxTime">
+<Heading>User preference <C>DownloadMaxTime</C></Heading>
+<Index Key="DownloadMaxTime"><C>DownloadMaxTime</C></Index>
+
+The value <C>0</C> (the default) means that no timeout is set
+in calls of <Ref Func="Download"/>.
+If the value is a positive integer <M>n</M> then those download methods that
+support a timeout will give up after <M>n</M> seconds.
+<P/>
+One can set the value of the preference to be <C>val</C> via
+<Ref Func="SetUserPreference" BookName="ref"/>, by calling
+<C>SetUserPreference( "utils", "DownloadMaxTime", val )</C>,
+and access the current value via
+<Ref Func="UserPreference" BookName="ref"/>, by calling
+<C>UserPreference( "utils", "DownloadMaxTime" )</C>.
+
+</Subsection>
+
 </Section>
 
 </Chapter>

lib/download.gd

@@ -34,3 +34,21 @@ curl or wget will omit the check of the server's certificate."
   package:= "utils",
   ) );
 
+
+#############################################################################
+##
+#U  DownloadMaxTime
+##
+DeclareUserPreference( rec(
+  name:= "DownloadMaxTime",
+  description:= [
+    "The value '0' (the default) means that no timeout is set \
+in calls of 'Download'. \
+If the value is a positive integer 'n' then those download methods that \
+support a timeout will give up after 'n' seconds."
+    ],
+  default:= 0,
+  check:= val -> val = 0 or IsPosInt( val ),
+  package:= "utils",
+  ) );
+

lib/download.gi

@@ -40,6 +40,7 @@ Add( Download_Methods, rec(
     if not IsBound( opt.failOnError ) then
       opt.failOnError:= true;
     fi;
+    # 'DownloadURL' handles the options 'verifyCert' and 'maxTime'.
     res:= ValueGlobal( "DownloadURL" )( url, opt );
 
     if res.success = true and
@@ -58,6 +59,8 @@ Add( Download_Methods, rec(
 
     if not StartsWith( url, "http://" ) then
       return rec( success:= false, error:= "protocol is not http" );
+    elif IsBound( opt.maxTime ) and opt.maxTime <> 0 then
+      return rec( success:= false, error:= "no support for given timeout" );
     fi;
 
     rurl:= ReplacedString( url, "http://", "" );
@@ -76,7 +79,8 @@ Add( Download_Methods, rec(
                   error:= res.status );
     elif res.statuscode >= 400 then
       return rec( success:= false,
-                  error:= Concatenation( "HTTP error code ", res.statuscode ) );
+                  error:= Concatenation( "HTTP error code ",
+                                         String( res.statuscode ) ) );
     elif not ( IsBound( opt.target ) and IsString( opt.target ) ) then
       return rec( success:= true, result:= res.body );
     else
@@ -95,6 +99,12 @@ Add( Download_Methods, rec(
   download:= function( url, opt )
     local res, outstream, exec, args, code;
 
+    if IsBound( opt.maxTime ) and opt.maxTime <> 0 then
+      # wget 1.20.3 ignores a given timeout.
+      # (wget 1.21.3 would support timeout.)
+      return rec( success:= false, error:= "no support for given timeout" );
+    fi;
+
     res:= "";
     outstream:= OutputTextString( res, true );
     exec:= Filename( DirectoriesSystemPrograms(), "wget" );
@@ -106,6 +116,9 @@ Add( Download_Methods, rec(
     if IsBound( opt.verifyCert ) and opt.verifyCert = false then
       Add( args, "--no-check-certificate" );
     fi;
+    if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then
+      Add( args, Concatenation( "--timeout=", String( opt.maxTime ) ) );
+    fi;
     code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args );
     CloseStream( outstream );
     if code <> 0 then
@@ -147,6 +160,10 @@ Add( Download_Methods, rec(
     else
       Add( args, "-" );
     fi;
+    if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then
+      Add( args, "--max-time" );
+      Add( args, opt.maxTime );
+    fi;
     Add( args, url );
     code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args );
     CloseStream( outstream );
@@ -177,14 +194,22 @@ InstallMethod( Download,
 InstallMethod( Download,
     [ "IsString", "IsRecord" ],
     function( url, opt )
-    local errors, r, res;
+    local timeout, errors, r, res;
 
     # Set the default for 'verifyCert' if necessary.
     if not IsBound( opt.verifyCert ) and
        UserPreference( "utils", "DownloadVerifyCertificate" ) = false then
       opt.verifyCert:= false;
     fi;
 
+    # Set the default for 'maxTime' if necessary.
+    if not IsBound( opt.maxTime ) then
+      timeout:= UserPreference( "utils", "DownloadMaxTime" );
+      if IsPosInt( timeout ) then
+        opt.maxTime:= timeout;
+      fi;
+    fi;
+
     # Run over the methods.
     errors:= [];
     for r in Download_Methods do
@@ -213,6 +238,11 @@ InstallMethod( Download,
       return rec( success:= false, error:= "no method was available" );
     else
       # At least one method was tried but without success.
+      if IsBound( opt.maxTime ) then
+        Add( errors,
+             Concatenation( "(maxTime option was set to ",
+                            String( opt.maxTime ), ")" ) );
+      fi;
       return rec( success:= false,
                   error:= JoinStringsWithSeparator( errors, "; " ) );
     fi;

tst/download.tst

@@ -1,4 +1,4 @@
-#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r;
+#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad
 ############################################################################
 ##
 #W  download.tst               Utils Package                   Thomas Breuer
@@ -54,8 +54,10 @@ gap> for pair in urls do
 >      if expected = false and Length( good2 ) > 0 then
 >        Print( "success for url ", url, "?\n" );
 >      fi;
->      if Length( good1 ) <> Length( good2 ) then
->        Print( "different success cases for url ", url, "\n" );
+>      if List( good1, x -> x[2] ) <> List( good2, x -> x[2] ) then
+>        Print( "different success cases for url ", url, ":\n",
+>               List( good1, x -> x[2] ), " vs. ", List( good2, x -> x[2] ),
+>               "\n" );
 >      fi;
 >      if Length( good1 ) > 0 then
 >        contents:= good1[1][1].result;
@@ -65,8 +67,36 @@ gap> for pair in urls do
 >          fi;
 >        od;
 >      fi;
+>      res3:= List( meths,
+>               r -> [ r.download( url,
+>                        rec( maxTime:= 10 ) ),
+>                      r.position ] );;
+>      good3:= Filtered( res3, r -> r[1].success = true );;
+>      if expected = false and Length( good3 ) > 0 then
+>        Print( "success for url ", url, "?\n" );
+>      fi;
+>      # The IO and wget based methods are available.
+>      # They cannot handle the 'maxTime' parameter.
+>      bad:= Filtered( meths,
+>                  x -> StartsWith( x.name, "via SingleHTTPRequest" ) or
+>                       StartsWith( x.name, "via wget" ) );
+>      bad:= List( bad, x -> x.position );
+>      good1:= Filtered( good1, x -> not x[2] in bad );
+>      if List( good1, x -> x[2] ) <> List( good3, x -> x[2] ) then
+>        Print( "different success cases for url ", url, ":\n",
+>               List( good1, x -> x[2] ), " vs. ", List( good3, x -> x[2] ),
+>               "\n" );
+>      fi;
 >    od;
 
+##  test timeout
+gap> res1:= Download( "https://httpbun.com/delay/3", rec( maxTime:= 1 ) );;
+gap> res1.success = false;
+true
+gap> res1:= Download( "https://httpbun.com/delay/3", rec( maxTime:= 5 ) );;
+gap> res1.success = true;
+true
+
 ##  the example 9.1.1 from the manual
 gap> url:= "https://www.gap-system.org/index.html";;
 gap> res1:= Download( url );;