diff --git a/src/plugin/protocol-okhttp/ivy.xml b/src/plugin/protocol-okhttp/ivy.xml index 0768def785..28f355d7b9 100644 --- a/src/plugin/protocol-okhttp/ivy.xml +++ b/src/plugin/protocol-okhttp/ivy.xml @@ -37,8 +37,9 @@ - - + + + - + diff --git a/src/plugin/protocol-okhttp/plugin.xml b/src/plugin/protocol-okhttp/plugin.xml index e2183d2b50..51f65f5d25 100755 --- a/src/plugin/protocol-okhttp/plugin.xml +++ b/src/plugin/protocol-okhttp/plugin.xml @@ -28,13 +28,15 @@ - - - - - - - + + + + + + + + + diff --git a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java index 954c3f6df1..a9d2b14d42 100644 --- a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java +++ b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java @@ -52,15 +52,19 @@ import org.slf4j.LoggerFactory; import okhttp3.Authenticator; +import okhttp3.CompressionInterceptor; import okhttp3.Connection; import okhttp3.ConnectionPool; +import okhttp3.Gzip; import okhttp3.Handshake; import okhttp3.Headers; import okhttp3.Interceptor; import okhttp3.OkHttpClient; import okhttp3.Protocol; import okhttp3.Request; -import okhttp3.brotli.BrotliInterceptor; +import okhttp3.brotli.Brotli; +import okhttp3.zstd.Zstd; + public class OkHttp extends HttpBase { @@ -156,13 +160,11 @@ public boolean verify(String hostname, SSLSession session) { String proxyUsername = conf.get("http.proxy.username"); if (proxyUsername == null) { ProxySelector selector = new ProxySelector() { - @SuppressWarnings("serial") private final List noProxyList = new ArrayList() { { add(Proxy.NO_PROXY); } }; - @SuppressWarnings("serial") private final List proxyList = new ArrayList() { { add(proxy); @@ -224,8 +226,9 @@ public Request authenticate(okhttp3.Route route, builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); } - // enable support for Brotli compression (Content-Encoding) - builder.addInterceptor(BrotliInterceptor.INSTANCE); + // enable support for Zstd, Brotli, Gzip Content-Encoding + builder.addInterceptor(new CompressionInterceptor(Zstd.INSTANCE, + Brotli.INSTANCE, Gzip.INSTANCE)); // instantiate connection pool(s), cf. // https://square.github.io/okhttp/3.x/okhttp/okhttp3/ConnectionPool.html