From 31492bfc1583fb2c43f1a30ed6b00955392ad81f Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sun, 18 Oct 2015 16:24:13 +0100 Subject: [PATCH 01/27] [bugfix] Make sure streams are closed --- .../httpclient/impl/ApacheHttpConnection.java | 19 ++++++++------- .../httpclient/impl/BinaryResponseBody.java | 3 +-- .../httpclient/impl/HrefRequestBody.java | 23 ++++++++----------- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/http-client-java/src/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/org/expath/httpclient/impl/ApacheHttpConnection.java index f22db54..5e29ad2 100644 --- a/http-client-java/src/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -9,10 +9,7 @@ package org.expath.httpclient.impl; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.net.ProxySelector; import java.net.URI; import org.apache.commons.logging.Log; @@ -36,6 +33,7 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.params.ClientPNames; import org.apache.http.conn.routing.HttpRoutePlanner; +import org.apache.http.entity.AbstractHttpEntity; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentProducer; import org.apache.http.entity.EntityTemplate; @@ -45,6 +43,7 @@ import org.apache.http.impl.conn.ProxySelectorRoutePlanner; import org.apache.http.params.HttpConnectionParams; import org.apache.http.params.HttpParams; +import org.apache.http.util.Args; import org.expath.httpclient.HeaderSet; import org.expath.httpclient.HttpClientException; import org.expath.httpclient.HttpConnection; @@ -353,7 +352,7 @@ private void setRequestEntity(HttpRequestBody body) return; } // make the entity from a new producer - HttpEntity entity; + final HttpEntity entity; if ( myVersion == HttpVersion.HTTP_1_1 ) { // Take advantage of HTTP 1.1 chunked encoding to stream the // payload directly to the request. @@ -366,10 +365,14 @@ private void setRequestEntity(HttpRequestBody body) // With HTTP 1.0, chunked encoding is not supported, so first // serialize into memory and use the resulting byte array as the // entity payload. - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - body.serialize(buffer); - entity = new ByteArrayEntity(buffer.toByteArray()); + try (final ByteArrayOutputStream buffer = new ByteArrayOutputStream()) { + body.serialize(buffer); + entity = new ByteArrayEntity(buffer.toByteArray()); + } catch (final IOException e) { + throw new HttpClientException(e.getMessage(), e); + } } + // cast the request HttpEntityEnclosingRequestBase req = null; if ( ! (myRequest instanceof HttpEntityEnclosingRequestBase) ) { diff --git a/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java b/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java index 06f12f4..3a96828 100644 --- a/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java +++ b/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java @@ -48,8 +48,7 @@ public BinaryResponseBody(Result result, InputStream in, ContentType type, Heade { myContentType = type; myHeaders = headers; - try { - ByteArrayOutputStream out = new ByteArrayOutputStream(); + try(final ByteArrayOutputStream out = new ByteArrayOutputStream()) { byte[] buf = new byte[4096]; int read = 0; while ( (read = in.read(buf)) > 0 ) { diff --git a/http-client-java/src/org/expath/httpclient/impl/HrefRequestBody.java b/http-client-java/src/org/expath/httpclient/impl/HrefRequestBody.java index fbcdb5a..34df06f 100644 --- a/http-client-java/src/org/expath/httpclient/impl/HrefRequestBody.java +++ b/http-client-java/src/org/expath/httpclient/impl/HrefRequestBody.java @@ -62,23 +62,18 @@ public void serialize(OutputStream out) { try { String filename = new URI(myHref).getPath(); - InputStream in = new FileInputStream(new File(filename)); - byte[] buf = new byte[4096]; - int l = -1; - while ( (l = in.read(buf)) != -1 ) { - out.write(buf, 0, l); + try (final InputStream in = new FileInputStream(new File(filename))) { + byte[] buf = new byte[4096]; + int l = -1; + while ((l = in.read(buf)) != -1) { + out.write(buf, 0, l); + } + } catch (IOException ex) { + throw new HttpClientException("Error sending the file content", ex); } - in.close(); - } - catch ( URISyntaxException ex ) { + } catch ( URISyntaxException ex ) { throw new HttpClientException("Bad URI: " + myHref, ex); } - catch ( FileNotFoundException ex ) { - throw new HttpClientException("Error sending the file content", ex); - } - catch ( IOException ex ) { - throw new HttpClientException("Error sending the file content", ex); - } } private String myHref; From 15b624aeeb47a3f29c35c5c448ebc7a40a04fe40 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sun, 18 Oct 2015 16:46:37 +0100 Subject: [PATCH 02/27] [refactor] When building results use Streams for Binary Content and Readers for Text Content --- .../httpclient/impl/BinaryResponseBody.java | 20 +------- .../httpclient/impl/TextResponseBody.java | 49 +++++-------------- .../httpclient/impl/XmlResponseBody.java | 21 +++----- .../org/expath/httpclient/model/Result.java | 14 +++--- .../expath/httpclient/saxon/SaxonResult.java | 38 +++++++++++--- 5 files changed, 60 insertions(+), 82 deletions(-) diff --git a/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java b/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java index 3a96828..f274da6 100644 --- a/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java +++ b/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java @@ -28,13 +28,6 @@ public class BinaryResponseBody implements HttpResponseBody { - public BinaryResponseBody(Result result, byte[] value, ContentType type, HeaderSet headers) - throws HttpClientException - { - myContentType = type; - myHeaders = headers; - result.add(value); - } // TODO: Work only for binary response. What if the response is encoded // with base64? @@ -48,18 +41,7 @@ public BinaryResponseBody(Result result, InputStream in, ContentType type, Heade { myContentType = type; myHeaders = headers; - try(final ByteArrayOutputStream out = new ByteArrayOutputStream()) { - byte[] buf = new byte[4096]; - int read = 0; - while ( (read = in.read(buf)) > 0 ) { - out.write(buf, 0, read); - } - byte[] bytes = out.toByteArray(); - result.add(bytes); - } - catch ( IOException ex ) { - throw new HttpClientException("error reading HTTP response", ex); - } + result.add(in); } @Override diff --git a/http-client-java/src/org/expath/httpclient/impl/TextResponseBody.java b/http-client-java/src/org/expath/httpclient/impl/TextResponseBody.java index 7c33c4c..b36c499 100644 --- a/http-client-java/src/org/expath/httpclient/impl/TextResponseBody.java +++ b/http-client-java/src/org/expath/httpclient/impl/TextResponseBody.java @@ -9,12 +9,10 @@ package org.expath.httpclient.impl; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.UnsupportedEncodingException; +import java.io.*; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + import org.expath.httpclient.ContentType; import org.expath.httpclient.HeaderSet; import org.expath.httpclient.HttpClientException; @@ -34,45 +32,20 @@ public class TextResponseBody public TextResponseBody(Result result, InputStream in, ContentType type, HeaderSet headers) throws HttpClientException { - // FIXME: ... - String charset = "utf-8"; - try { - Reader reader = new InputStreamReader(in, charset); - init(result, reader, type, headers); - } - catch ( UnsupportedEncodingException ex ) { - String msg = "not supported charset reading HTTP response: " + charset; - throw new HttpClientException(msg, ex); - } + myContentType = type; + myHeaders = headers; + // TODO: ... + final Charset charset = StandardCharsets.UTF_8; + final Reader reader = new InputStreamReader(in, charset); + result.add(reader); } public TextResponseBody(Result result, Reader in, ContentType type, HeaderSet headers) throws HttpClientException - { - init(result, in, type, headers); - } - - private void init(Result result, Reader in, ContentType type, HeaderSet headers) - throws HttpClientException { myContentType = type; myHeaders = headers; - // BufferedReader handles the ends of line (all \n, \r, and \r\n are - // transformed to \n) - try { - StringBuilder builder = new StringBuilder(); - BufferedReader buf_in = new BufferedReader(in); - String buf = null; - while ( (buf = buf_in.readLine()) != null ) { - builder.append(buf); - builder.append('\n'); - } - String value = builder.toString(); - result.add(value); - } - catch ( IOException ex ) { - throw new HttpClientException("error reading HTTP response", ex); - } + result.add(in); } @Override diff --git a/http-client-java/src/org/expath/httpclient/impl/XmlResponseBody.java b/http-client-java/src/org/expath/httpclient/impl/XmlResponseBody.java index 6473876..4a65e16 100644 --- a/http-client-java/src/org/expath/httpclient/impl/XmlResponseBody.java +++ b/http-client-java/src/org/expath/httpclient/impl/XmlResponseBody.java @@ -13,6 +13,8 @@ import java.io.InputStreamReader; import java.io.Reader; import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import javax.xml.transform.Source; import javax.xml.transform.sax.SAXSource; import javax.xml.transform.stream.StreamSource; @@ -39,15 +41,9 @@ public XmlResponseBody(Result result, InputStream in, ContentType type, HeaderSe throws HttpClientException { // TODO: ... - String charset = "utf-8"; - try { - Reader reader = new InputStreamReader(in, charset); - init(result, reader, type, headers, html); - } - catch ( UnsupportedEncodingException ex ) { - String msg = "not supported charset reading HTTP response: " + charset; - throw new HttpClientException(msg, ex); - } + final Charset charset = StandardCharsets.UTF_8; + final Reader reader = new InputStreamReader(in, charset); + init(result, reader, type, headers, html); } public XmlResponseBody(Result result, Reader in, ContentType type, HeaderSet headers, boolean html) @@ -56,7 +52,7 @@ public XmlResponseBody(Result result, Reader in, ContentType type, HeaderSet hea init(result, in, type, headers, html); } - private void init(Result result, Reader in, ContentType type, HeaderSet headers, boolean html) + private void init(final Result result, final Reader in, final ContentType type, final HeaderSet headers, final boolean html) throws HttpClientException { myContentType = type; @@ -71,13 +67,12 @@ private void init(Result result, Reader in, ContentType type, HeaderSet headers, InputSource input = new InputSource(in); src = new SAXSource(parser, input); src.setSystemId(sys_id); - } - else { + } else { src = new StreamSource(in, sys_id); } result.add(src); } - catch ( SAXException ex ) { + catch (SAXException ex) { throw new HttpClientException("error parsing result HTML", ex); } } diff --git a/http-client-java/src/org/expath/httpclient/model/Result.java b/http-client-java/src/org/expath/httpclient/model/Result.java index c16dc9f..cd1c135 100644 --- a/http-client-java/src/org/expath/httpclient/model/Result.java +++ b/http-client-java/src/org/expath/httpclient/model/Result.java @@ -9,6 +9,8 @@ package org.expath.httpclient.model; +import java.io.Reader; +import java.io.InputStream; import javax.xml.transform.Source; import org.expath.httpclient.HttpClientException; import org.expath.httpclient.HttpResponse; @@ -63,22 +65,22 @@ public Result makeNewResult() throws HttpClientException; /** - * Add an {@code xs:string} to the result sequence. + * Add a string value to the result sequence. * * @param string The string to add to the result sequence. * @throws HttpClientException If any error occurs. */ - public void add(String string) + public void add(Reader string) throws HttpClientException; /** - * Add an {@code xs:base64Binary} to the result sequence. + * Add raw binary to the result sequence. * - * @param bytes The bytes representing the base64 binary item to add to the + * @param bytes The bytes representing the binary item to add to the * result sequence. * @throws HttpClientException If any error occurs. */ - public void add(byte[] bytes) + public void add(InputStream bytes) throws HttpClientException; /** @@ -124,5 +126,5 @@ public void add(HttpResponse response) /* */ /* The Initial Developer of the Original Code is Florent Georges. */ /* */ -/* Contributor(s): none. */ +/* Contributor(s): Adam Retter */ /* ------------------------------------------------------------------------ */ diff --git a/http-client-saxon/src/org/expath/httpclient/saxon/SaxonResult.java b/http-client-saxon/src/org/expath/httpclient/saxon/SaxonResult.java index 14f71a9..34eeb28 100644 --- a/http-client-saxon/src/org/expath/httpclient/saxon/SaxonResult.java +++ b/http-client-saxon/src/org/expath/httpclient/saxon/SaxonResult.java @@ -9,6 +9,7 @@ package org.expath.httpclient.saxon; +import java.io.*; import java.util.ArrayList; import java.util.List; import javax.xml.transform.Source; @@ -50,19 +51,44 @@ public Result makeNewResult() } @Override - public void add(String string) + public void add(Reader reader) throws HttpClientException { - Item item = new StringValue(string); - myItems.add(item); + try(final BufferedReader buf_in = new BufferedReader(reader)) { + final StringBuilder builder = new StringBuilder(); + + String buf = null; + while ( (buf = buf_in.readLine()) != null ) { + builder.append(buf); + builder.append('\n'); + } + final String value = builder.toString(); + + Item item = new StringValue(value); + myItems.add(item); + } + catch ( final IOException ex ) { + throw new HttpClientException("error reading HTTP response", ex); + } } @Override - public void add(byte[] bytes) + public void add(InputStream inputStream) throws HttpClientException { - Item item = new Base64BinaryValue(bytes); - myItems.add(item); + try(final ByteArrayOutputStream out = new ByteArrayOutputStream()) { + final byte[] buf = new byte[4096]; + int read = -1; + while ( (read = inputStream.read(buf)) > 0 ) { + out.write(buf, 0, read); + } + final byte[] bytes = out.toByteArray(); + + Item item = new Base64BinaryValue(bytes); + myItems.add(item); + } catch(final IOException e) { + throw new HttpClientException(e.getMessage(), e); + } } @Override From d59b340fdf248ec6650b729c45276a061fc83c7c Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sun, 18 Oct 2015 12:50:49 +0100 Subject: [PATCH 03/27] [feature] Maven POM files for EXPath HTTP Client --- .gitignore | 3 ++ http-client-java/pom.xml | 65 ++++++++++++++++++++++++++++++++++++++ http-client-parent/pom.xml | 50 +++++++++++++++++++++++++++++ http-client-saxon/pom.xml | 61 +++++++++++++++++++++++++++++++++++ pom.xml | 26 +++++++++++++++ 5 files changed, 205 insertions(+) create mode 100644 http-client-java/pom.xml create mode 100644 http-client-parent/pom.xml create mode 100644 http-client-saxon/pom.xml create mode 100644 pom.xml diff --git a/.gitignore b/.gitignore index dc94cc4..a961cfc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ /http-client-saxon/build/ /http-client-saxon/dist/ /http-client-saxon/src/org/expath/httpclient/saxon/version.properties + +/http-client-java/target/ +/http-client-saxon/target/ diff --git a/http-client-java/pom.xml b/http-client-java/pom.xml new file mode 100644 index 0000000..f255b52 --- /dev/null +++ b/http-client-java/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + + org.expath.http.client + http-client-parent + 1.0-SNAPSHOT + ../http-client-parent/pom.xml + + + http-client-java + jar + + EXPath HTTP Client Java Library + Java Library implementing the core HTTP Client module features + + + scm:git:git://github.com/adamretter/expath-http-client-java.git + scm:git:git://github.com/adamretter/expath-http-client-java.git + http://github.com/adamretter/expath-http-client-java + + + + + org.expath.tools + tools-java + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + + + org.ccil.cowan.tagsoup + tagsoup + 1.2.1 + + + org.apache.james + apache-mime4j-core + 0.8.1 + + + org.apache.james + apache-mime4j-dom + 0.8.1 + + + commons-logging + commons-logging + + + + junit + junit + test + + + + diff --git a/http-client-parent/pom.xml b/http-client-parent/pom.xml new file mode 100644 index 0000000..0c3fd8f --- /dev/null +++ b/http-client-parent/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + + org.expath + expath-parent + 1.0-SNAPSHOT + ../../expath-parent/pom.xml + + + org.expath.http.client + http-client-parent + pom + + EXPath HTTP Client POM + The base POM for all EXPath HTTP Client modules. + + + scm:git:git://github.com/adamretter/expath-http-client-java.git + scm:git:git://github.com/adamretter/expath-http-client-java.git + http://github.com/adamretter/expath-http-client-java + + + + 1.0-SNAPSHOT + + + + + + org.expath.tools + tools-java + 1.0-SNAPSHOT + + + org.apache.httpcomponents + httpcore + 4.4.6 + + + org.apache.httpcomponents + httpclient + 4.5.3 + + + + + diff --git a/http-client-saxon/pom.xml b/http-client-saxon/pom.xml new file mode 100644 index 0000000..3c139be --- /dev/null +++ b/http-client-saxon/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + + org.expath.http.client + http-client-parent + 1.0-SNAPSHOT + ../http-client-parent/pom.xml + + + http-client-saxon + jar + + EXPath HTTP Client Saxon Implementation + Implementation of HTTP Client module for Saxon + + + scm:git:git://github.com/adamretter/expath-http-client-java.git + scm:git:git://github.com/adamretter/expath-http-client-java.git + http://github.com/adamretter/expath-http-client-java + + + + + org.expath.tools + tools-java + + + org.expath.tools + tools-saxon + ${expath.tools.version} + + + ${project.groupId} + http-client-java + ${project.version} + + + org.expath.packaging + pkg-saxon + 1.0-SNAPSHOT + + + net.sf.saxon + Saxon-HE + + + org.apache.httpcomponents + httpcore + + + + junit + junit + test + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..7a0c9d7 --- /dev/null +++ b/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + + org.expath.http.client + http-client-parent + 1.0-SNAPSHOT + http-client-parent/pom.xml + + + http-client + pom + + EXPath HTTP Client + + EXPath HTTP Client module. + + + http-client-parent + http-client-java + http-client-saxon + + + \ No newline at end of file From e759009c611f05e12979b87333050745f5e7188a Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sun, 18 Oct 2015 12:51:21 +0100 Subject: [PATCH 04/27] [refatcor] Move files to standard project layout --- http-client-java/src/{ => main/java}/net/iharder/Base64.java | 0 .../src/{ => main/java}/org/expath/httpclient/ContentType.java | 0 .../src/{ => main/java}/org/expath/httpclient/HeaderSet.java | 0 .../src/{ => main/java}/org/expath/httpclient/HttpClient.java | 0 .../java}/org/expath/httpclient/HttpClientException.java | 0 .../src/{ => main/java}/org/expath/httpclient/HttpConnection.java | 0 .../src/{ => main/java}/org/expath/httpclient/HttpConstants.java | 0 .../{ => main/java}/org/expath/httpclient/HttpCredentials.java | 0 .../src/{ => main/java}/org/expath/httpclient/HttpRequest.java | 0 .../{ => main/java}/org/expath/httpclient/HttpRequestBody.java | 0 .../src/{ => main/java}/org/expath/httpclient/HttpResponse.java | 0 .../{ => main/java}/org/expath/httpclient/HttpResponseBody.java | 0 .../java}/org/expath/httpclient/impl/AnyEmptyMethod.java | 0 .../java}/org/expath/httpclient/impl/AnyEntityMethod.java | 0 .../java}/org/expath/httpclient/impl/ApacheHttpConnection.java | 0 .../java}/org/expath/httpclient/impl/BinaryResponseBody.java | 0 .../{ => main/java}/org/expath/httpclient/impl/BodyFactory.java | 0 .../java}/org/expath/httpclient/impl/HrefRequestBody.java | 0 .../java}/org/expath/httpclient/impl/HttpRequestImpl.java | 0 .../{ => main/java}/org/expath/httpclient/impl/LoggerHelper.java | 0 .../java}/org/expath/httpclient/impl/MultipartRequestBody.java | 0 .../java}/org/expath/httpclient/impl/MultipartResponseBody.java | 0 .../{ => main/java}/org/expath/httpclient/impl/RequestParser.java | 0 .../java}/org/expath/httpclient/impl/SinglePartRequestBody.java | 0 .../java}/org/expath/httpclient/impl/TextResponseBody.java | 0 .../java}/org/expath/httpclient/impl/XmlResponseBody.java | 0 .../src/{ => main/java}/org/expath/httpclient/model/Result.java | 0 .../{ => main/java}/org/expath/httpclient/model/TreeBuilder.java | 0 .../{test => src/test/java}/tmp/tests/ApacheHttpClientTest.java | 0 .../{ => main/java}/org/expath/httpclient/saxon/SaxonResult.java | 0 .../java}/org/expath/httpclient/saxon/SaxonTreeBuilder.java | 0 .../java}/org/expath/httpclient/saxon/SendRequestCall.java | 0 .../java}/org/expath/httpclient/saxon/SendRequestFunction.java | 0 .../{test => src/test/java}/org/fgeorges/google/GContactTest.java | 0 34 files changed, 0 insertions(+), 0 deletions(-) rename http-client-java/src/{ => main/java}/net/iharder/Base64.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/ContentType.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HeaderSet.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpClient.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpClientException.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpConnection.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpConstants.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpCredentials.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpRequest.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpRequestBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpResponse.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/HttpResponseBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/AnyEmptyMethod.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/AnyEntityMethod.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/ApacheHttpConnection.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/BinaryResponseBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/BodyFactory.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/HrefRequestBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/HttpRequestImpl.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/LoggerHelper.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/MultipartRequestBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/MultipartResponseBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/RequestParser.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/SinglePartRequestBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/TextResponseBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/impl/XmlResponseBody.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/model/Result.java (100%) rename http-client-java/src/{ => main/java}/org/expath/httpclient/model/TreeBuilder.java (100%) rename http-client-java/{test => src/test/java}/tmp/tests/ApacheHttpClientTest.java (100%) rename http-client-saxon/src/{ => main/java}/org/expath/httpclient/saxon/SaxonResult.java (100%) rename http-client-saxon/src/{ => main/java}/org/expath/httpclient/saxon/SaxonTreeBuilder.java (100%) rename http-client-saxon/src/{ => main/java}/org/expath/httpclient/saxon/SendRequestCall.java (100%) rename http-client-saxon/src/{ => main/java}/org/expath/httpclient/saxon/SendRequestFunction.java (100%) rename http-client-saxon/{test => src/test/java}/org/fgeorges/google/GContactTest.java (100%) diff --git a/http-client-java/src/net/iharder/Base64.java b/http-client-java/src/main/java/net/iharder/Base64.java similarity index 100% rename from http-client-java/src/net/iharder/Base64.java rename to http-client-java/src/main/java/net/iharder/Base64.java diff --git a/http-client-java/src/org/expath/httpclient/ContentType.java b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/ContentType.java rename to http-client-java/src/main/java/org/expath/httpclient/ContentType.java diff --git a/http-client-java/src/org/expath/httpclient/HeaderSet.java b/http-client-java/src/main/java/org/expath/httpclient/HeaderSet.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HeaderSet.java rename to http-client-java/src/main/java/org/expath/httpclient/HeaderSet.java diff --git a/http-client-java/src/org/expath/httpclient/HttpClient.java b/http-client-java/src/main/java/org/expath/httpclient/HttpClient.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpClient.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpClient.java diff --git a/http-client-java/src/org/expath/httpclient/HttpClientException.java b/http-client-java/src/main/java/org/expath/httpclient/HttpClientException.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpClientException.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpClientException.java diff --git a/http-client-java/src/org/expath/httpclient/HttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpConnection.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java diff --git a/http-client-java/src/org/expath/httpclient/HttpConstants.java b/http-client-java/src/main/java/org/expath/httpclient/HttpConstants.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpConstants.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpConstants.java diff --git a/http-client-java/src/org/expath/httpclient/HttpCredentials.java b/http-client-java/src/main/java/org/expath/httpclient/HttpCredentials.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpCredentials.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpCredentials.java diff --git a/http-client-java/src/org/expath/httpclient/HttpRequest.java b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpRequest.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java diff --git a/http-client-java/src/org/expath/httpclient/HttpRequestBody.java b/http-client-java/src/main/java/org/expath/httpclient/HttpRequestBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpRequestBody.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpRequestBody.java diff --git a/http-client-java/src/org/expath/httpclient/HttpResponse.java b/http-client-java/src/main/java/org/expath/httpclient/HttpResponse.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpResponse.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpResponse.java diff --git a/http-client-java/src/org/expath/httpclient/HttpResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/HttpResponseBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/HttpResponseBody.java rename to http-client-java/src/main/java/org/expath/httpclient/HttpResponseBody.java diff --git a/http-client-java/src/org/expath/httpclient/impl/AnyEmptyMethod.java b/http-client-java/src/main/java/org/expath/httpclient/impl/AnyEmptyMethod.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/AnyEmptyMethod.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/AnyEmptyMethod.java diff --git a/http-client-java/src/org/expath/httpclient/impl/AnyEntityMethod.java b/http-client-java/src/main/java/org/expath/httpclient/impl/AnyEntityMethod.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/AnyEntityMethod.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/AnyEntityMethod.java diff --git a/http-client-java/src/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/ApacheHttpConnection.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java diff --git a/http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/BinaryResponseBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/BinaryResponseBody.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/BinaryResponseBody.java diff --git a/http-client-java/src/org/expath/httpclient/impl/BodyFactory.java b/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/BodyFactory.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java diff --git a/http-client-java/src/org/expath/httpclient/impl/HrefRequestBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/HrefRequestBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/HrefRequestBody.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/HrefRequestBody.java diff --git a/http-client-java/src/org/expath/httpclient/impl/HttpRequestImpl.java b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/HttpRequestImpl.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java diff --git a/http-client-java/src/org/expath/httpclient/impl/LoggerHelper.java b/http-client-java/src/main/java/org/expath/httpclient/impl/LoggerHelper.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/LoggerHelper.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/LoggerHelper.java diff --git a/http-client-java/src/org/expath/httpclient/impl/MultipartRequestBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartRequestBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/MultipartRequestBody.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/MultipartRequestBody.java diff --git a/http-client-java/src/org/expath/httpclient/impl/MultipartResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/MultipartResponseBody.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java diff --git a/http-client-java/src/org/expath/httpclient/impl/RequestParser.java b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/RequestParser.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java diff --git a/http-client-java/src/org/expath/httpclient/impl/SinglePartRequestBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/SinglePartRequestBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/SinglePartRequestBody.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/SinglePartRequestBody.java diff --git a/http-client-java/src/org/expath/httpclient/impl/TextResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/TextResponseBody.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java diff --git a/http-client-java/src/org/expath/httpclient/impl/XmlResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/impl/XmlResponseBody.java rename to http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java diff --git a/http-client-java/src/org/expath/httpclient/model/Result.java b/http-client-java/src/main/java/org/expath/httpclient/model/Result.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/model/Result.java rename to http-client-java/src/main/java/org/expath/httpclient/model/Result.java diff --git a/http-client-java/src/org/expath/httpclient/model/TreeBuilder.java b/http-client-java/src/main/java/org/expath/httpclient/model/TreeBuilder.java similarity index 100% rename from http-client-java/src/org/expath/httpclient/model/TreeBuilder.java rename to http-client-java/src/main/java/org/expath/httpclient/model/TreeBuilder.java diff --git a/http-client-java/test/tmp/tests/ApacheHttpClientTest.java b/http-client-java/src/test/java/tmp/tests/ApacheHttpClientTest.java similarity index 100% rename from http-client-java/test/tmp/tests/ApacheHttpClientTest.java rename to http-client-java/src/test/java/tmp/tests/ApacheHttpClientTest.java diff --git a/http-client-saxon/src/org/expath/httpclient/saxon/SaxonResult.java b/http-client-saxon/src/main/java/org/expath/httpclient/saxon/SaxonResult.java similarity index 100% rename from http-client-saxon/src/org/expath/httpclient/saxon/SaxonResult.java rename to http-client-saxon/src/main/java/org/expath/httpclient/saxon/SaxonResult.java diff --git a/http-client-saxon/src/org/expath/httpclient/saxon/SaxonTreeBuilder.java b/http-client-saxon/src/main/java/org/expath/httpclient/saxon/SaxonTreeBuilder.java similarity index 100% rename from http-client-saxon/src/org/expath/httpclient/saxon/SaxonTreeBuilder.java rename to http-client-saxon/src/main/java/org/expath/httpclient/saxon/SaxonTreeBuilder.java diff --git a/http-client-saxon/src/org/expath/httpclient/saxon/SendRequestCall.java b/http-client-saxon/src/main/java/org/expath/httpclient/saxon/SendRequestCall.java similarity index 100% rename from http-client-saxon/src/org/expath/httpclient/saxon/SendRequestCall.java rename to http-client-saxon/src/main/java/org/expath/httpclient/saxon/SendRequestCall.java diff --git a/http-client-saxon/src/org/expath/httpclient/saxon/SendRequestFunction.java b/http-client-saxon/src/main/java/org/expath/httpclient/saxon/SendRequestFunction.java similarity index 100% rename from http-client-saxon/src/org/expath/httpclient/saxon/SendRequestFunction.java rename to http-client-saxon/src/main/java/org/expath/httpclient/saxon/SendRequestFunction.java diff --git a/http-client-saxon/test/org/fgeorges/google/GContactTest.java b/http-client-saxon/src/test/java/org/fgeorges/google/GContactTest.java similarity index 100% rename from http-client-saxon/test/org/fgeorges/google/GContactTest.java rename to http-client-saxon/src/test/java/org/fgeorges/google/GContactTest.java From d11dfed4e073aa66d40cd8920cd117b2dbbc656f Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 17 Aug 2017 08:03:39 -0400 Subject: [PATCH 05/27] [bugfix] Fix the compilation with Mime4j 0.8.1 --- .../impl/MultipartResponseBody.java | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java index 6e0f783..f4a4984 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List; import org.apache.commons.logging.Log; @@ -177,29 +178,33 @@ private HttpResponseBody makeResponsePart(Result result, HeaderSet headers, Mime throw new HttpClientException("impossible to find the content type"); } ContentType type = new ContentType(h); - switch ( BodyFactory.parseType(type) ) { - case XML: { - // TODO: 'content_type' is the header Content-Type without any - // param (i.e. "text/xml".) Should we keep this, or put the - // whole header (i.e. "text/xml; charset=utf-8")? (and for - // other types as well...) - Reader in = parser.getReader(); - return new XmlResponseBody(result, in, type, headers, false); - } - case HTML: { - Reader in = parser.getReader(); - return new XmlResponseBody(result, in, type, headers, true); - } - case TEXT: { - Reader in = parser.getReader(); - return new TextResponseBody(result, in, type, headers); - } - case BINARY: { - InputStream in = parser.getInputStream(); - return new BinaryResponseBody(result, in, type, headers); + try { + switch (BodyFactory.parseType(type)) { + case XML: { + // TODO: 'content_type' is the header Content-Type without any + // param (i.e. "text/xml".) Should we keep this, or put the + // whole header (i.e. "text/xml; charset=utf-8")? (and for + // other types as well...) + Reader in = parser.getReader(); + return new XmlResponseBody(result, in, type, headers, false); + } + case HTML: { + Reader in = parser.getReader(); + return new XmlResponseBody(result, in, type, headers, true); + } + case TEXT: { + Reader in = parser.getReader(); + return new TextResponseBody(result, in, type, headers); + } + case BINARY: { + InputStream in = parser.getInputStream(); + return new BinaryResponseBody(result, in, type, headers); + } + default: + throw new HttpClientException("INTERNAL ERROR: cannot happen"); } - default: - throw new HttpClientException("INTERNAL ERROR: cannot happen"); + } catch (UnsupportedEncodingException ex) { + throw new HttpClientException("Unable to parse response part", ex); } } From 1ff2593d7acb52ac5b69c4007ec1edcda4ba1fbc Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sun, 18 Oct 2015 13:32:34 +0100 Subject: [PATCH 06/27] [feature] Added an assembly for the EXPath HTTP Java Client --- http-client-java/pom.xml | 21 +++++++++++++++++++ .../src/main/assembly/assembly.xml | 19 +++++++++++++++++ http-client-parent/pom.xml | 11 ++++++++++ 3 files changed, 51 insertions(+) create mode 100644 http-client-java/src/main/assembly/assembly.xml diff --git a/http-client-java/pom.xml b/http-client-java/pom.xml index f255b52..ff5ce53 100644 --- a/http-client-java/pom.xml +++ b/http-client-java/pom.xml @@ -22,6 +22,27 @@ http://github.com/adamretter/expath-http-client-java + + + + maven-assembly-plugin + + + package + + single + + + + + + src/main/assembly/assembly.xml + + + + + + org.expath.tools diff --git a/http-client-java/src/main/assembly/assembly.xml b/http-client-java/src/main/assembly/assembly.xml new file mode 100644 index 0000000..001c3d9 --- /dev/null +++ b/http-client-java/src/main/assembly/assembly.xml @@ -0,0 +1,19 @@ + + + application + + dir + + + true + + + + / + true + runtime + + + \ No newline at end of file diff --git a/http-client-parent/pom.xml b/http-client-parent/pom.xml index 0c3fd8f..496d4ad 100644 --- a/http-client-parent/pom.xml +++ b/http-client-parent/pom.xml @@ -26,6 +26,17 @@ 1.0-SNAPSHOT + + + + + + maven-assembly-plugin + 2.6 + + + + From e0e38eacb5ca20f4e3a80db36a9611b679dd5daf Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 17 Aug 2017 07:32:25 -0400 Subject: [PATCH 07/27] [bugfix] Ensure chunked transfer encoding for HTTP 1.1 --- .../java/org/expath/httpclient/impl/ApacheHttpConnection.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 5e29ad2..772add2 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -299,6 +299,7 @@ private AbstractHttpClient makeClient() HttpConnectionParams.setConnectionTimeout(params, myTimeout * 1000); HttpConnectionParams.setSoTimeout(params, myTimeout * 1000); } + // the shared cookie store client.setCookieStore(COOKIES); // the HTTP version (1.0 or 1.1) @@ -359,6 +360,7 @@ private void setRequestEntity(HttpRequestBody body) ContentProducer producer = new RequestBodyProducer(body); EntityTemplate template = new EntityTemplate(producer); template.setContentType(body.getContentType()); + template.setChunked(true); entity = template; } else { From f392ef962c437bc547c855c202861bb0b7560bd9 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 17 Aug 2017 11:29:55 -0400 Subject: [PATCH 08/27] [bugfix] Avoid "null" in some error messages, as the message from Apache HTTP Client may be on the exception cause --- .../httpclient/impl/ApacheHttpConnection.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 772add2..d7bbce6 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -103,10 +103,33 @@ public void connect(HttpRequestBody body, HttpCredentials cred) } } catch ( IOException ex ) { - throw new HttpClientException("Error executing the HTTP method: " + ex.getMessage(), ex); + final String message = getMessage(ex); + throw new HttpClientException("Error executing the HTTP method: " + message != null ? message : "", ex); } } + /** + * Retrieves a message from the Throwable + * or its cause (recursively). + * + * @param throwable A thrown exception + * + * @return The first message, or null if there are no messages + * at all. + */ + private String getMessage(final Throwable throwable) { + if(throwable.getMessage() != null) { + return throwable.getMessage(); + } + + final Throwable cause = throwable.getCause(); + if(cause == null || cause == throwable) { + return null; + } + + return getMessage(cause); + } + public void disconnect() { if ( myClient != null ) { From f9adeb2fede6a67637575554d85538afcb47a78c Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 17 Aug 2017 16:06:57 -0400 Subject: [PATCH 09/27] [bugfix] It is an error if the user manually sets the Content-Type or Transfer-Encoding headers as these will be determined by Apache HTTP Client --- .../java/org/expath/httpclient/impl/RequestParser.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java index ccca30a..babb5ce 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java @@ -217,6 +217,15 @@ else if ( "value".equals(local) ) { if ( name == null || value == null ) { throw new HttpClientException("@name and @value are required on http:header"); } + + if(name.equalsIgnoreCase("Content-Length")) { + throw new HttpClientException("Content-Length should not be explicitly provided, either it will automatically be added or Transfer-Encoding will be used."); + } + + if(name.equalsIgnoreCase("Transfer-Encoding")) { + throw new HttpClientException("Transfer-Encoding should not be explicitly provided, it will automatically be added if required."); + } + // actually add the header headers.add(name, value); } From 4a2c8d1cd661ae7a939f53782804fcfb95867b27 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 17 Aug 2017 16:07:47 -0400 Subject: [PATCH 10/27] [refactor] Switched away from using deprecated Apache HTTP Components methods --- http-client-java/pom.xml | 5 +- .../httpclient/impl/ApacheHttpConnection.java | 120 ++++---- .../java/tmp/tests/ApacheHttpClientTest.java | 276 ++++++++++-------- 3 files changed, 221 insertions(+), 180 deletions(-) diff --git a/http-client-java/pom.xml b/http-client-java/pom.xml index ff5ce53..7909d5b 100644 --- a/http-client-java/pom.xml +++ b/http-client-java/pom.xml @@ -75,7 +75,10 @@ commons-logging commons-logging - + + net.jcip + jcip-annotations + junit junit diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index d7bbce6..6b58c67 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -10,8 +10,9 @@ package org.expath.httpclient.impl; import java.io.*; -import java.net.ProxySelector; import java.net.URI; + +import net.jcip.annotations.NotThreadSafe; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.Header; @@ -22,28 +23,15 @@ import org.apache.http.auth.Credentials; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CookieStore; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpOptions; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpTrace; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.params.ClientPNames; -import org.apache.http.conn.routing.HttpRoutePlanner; -import org.apache.http.entity.AbstractHttpEntity; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.*; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentProducer; import org.apache.http.entity.EntityTemplate; -import org.apache.http.impl.client.AbstractHttpClient; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.ProxySelectorRoutePlanner; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; -import org.apache.http.util.Args; +import org.apache.http.impl.client.*; +import org.apache.http.impl.conn.*; import org.expath.httpclient.HeaderSet; import org.expath.httpclient.HttpClientException; import org.expath.httpclient.HttpConnection; @@ -56,6 +44,7 @@ * * @author Florent Georges */ +@NotThreadSafe public class ApacheHttpConnection implements HttpConnection { @@ -65,7 +54,6 @@ public ApacheHttpConnection(URI uri) myRequest = null; myResponse = null; myVersion = DEFAULT_HTTP_VERSION; - myClient = null; } public void connect(HttpRequestBody body, HttpCredentials cred) @@ -74,11 +62,14 @@ public void connect(HttpRequestBody body, HttpCredentials cred) if ( myRequest == null ) { throw new HttpClientException("setRequestMethod has not been called before"); } + + myRequest.setProtocolVersion(myVersion); + try { // make a new client - myClient = makeClient(); + final CloseableHttpClient myClient = makeClient(); // set the credentials (if any) - setCredentials(cred); + final HttpClientContext clientContext = setCredentials(cred); // set the request entity body (if any) setRequestEntity(body); // log the request headers? @@ -89,7 +80,7 @@ public void connect(HttpRequestBody body, HttpCredentials cred) LoggerHelper.logCookies(LOG, "COOKIES", COOKIES.getCookies()); } // send the request - myResponse = myClient.execute(myRequest); + myResponse = myClient.execute(myRequest, clientContext); // TODO: Handle 'Connection' headers (for instance "Connection: close") // See for instance http://www.jmarshall.com/easy/http/. @@ -105,6 +96,8 @@ public void connect(HttpRequestBody body, HttpCredentials cred) catch ( IOException ex ) { final String message = getMessage(ex); throw new HttpClientException("Error executing the HTTP method: " + message != null ? message : "", ex); + } finally { + state = State.POST_CONNECT; } } @@ -130,17 +123,16 @@ private String getMessage(final Throwable throwable) { return getMessage(cause); } - public void disconnect() - { - if ( myClient != null ) { - myClient.getConnectionManager().shutdown(); - } + @Override + public void disconnect() { + clientConnectionManager.shutdown(); } - public void setHttpVersion(String ver) + @Override + public void setHttpVersion(final String ver) throws HttpClientException { - if ( myClient != null ) { + if ( state != State.INITIAL ) { String msg = "Internal error, HTTP version cannot been " + "set after connect() has been called."; throw new HttpClientException(msg); @@ -305,41 +297,48 @@ public InputStream getResponseStream() /** * Make a new Apache HTTP client, in order to serve this request. */ - private AbstractHttpClient makeClient() - { - AbstractHttpClient client = new DefaultHttpClient(); - HttpParams params = client.getParams(); + private CloseableHttpClient makeClient() { + + final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); + clientBuilder.setConnectionManager(clientConnectionManager); + // use the default JVM proxy settings (http.proxyHost, etc.) - HttpRoutePlanner route = new ProxySelectorRoutePlanner( - client.getConnectionManager().getSchemeRegistry(), - ProxySelector.getDefault()); - client.setRoutePlanner(route); + clientBuilder.setRoutePlanner(new SystemDefaultRoutePlanner(null)); + // do follow redirections? - params.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, myFollowRedirect); + if(myFollowRedirect) { + clientBuilder.setRedirectStrategy(LaxRedirectStrategy.INSTANCE); + } + + // the shared cookie store + clientBuilder.setDefaultCookieStore(COOKIES); + // set the timeout if any - if ( myTimeout != null ) { + final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); + if(myTimeout != null) { // See http://blog.jayway.com/2009/03/17/configuring-timeout-with-apache-httpclient-40/ - HttpConnectionParams.setConnectionTimeout(params, myTimeout * 1000); - HttpConnectionParams.setSoTimeout(params, myTimeout * 1000); + requestConfigBuilder + .setConnectTimeout(myTimeout * 1000) + .setSocketTimeout(myTimeout * 1000); } + clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); - // the shared cookie store - client.setCookieStore(COOKIES); - // the HTTP version (1.0 or 1.1) - params.setParameter("http.protocol.version", myVersion); - // return the just built client + final CloseableHttpClient client = clientBuilder.build(); return client; } /** * Set the credentials on the client, based on the {@link HttpCredentials} object. */ - private void setCredentials(HttpCredentials cred) + private HttpClientContext setCredentials(HttpCredentials cred) throws HttpClientException { + final HttpClientContext clientContext = HttpClientContext.create(); + if ( cred == null ) { - return; + return clientContext; } + URI uri = myRequest.getURI(); int port = uri.getPort(); if ( port == -1 ) { @@ -361,9 +360,12 @@ else if ( "https".equals(scheme) ) { LOG.debug("Set credentials for " + host + ":" + port + " - " + user + " - ***"); } - Credentials c = new UsernamePasswordCredentials(user, pwd); - AuthScope scope = new AuthScope(host, port); - myClient.getCredentialsProvider().setCredentials(scope, c); + + final Credentials c = new UsernamePasswordCredentials(user, pwd); + final AuthScope scope = new AuthScope(host, port); + + clientContext.getCredentialsProvider().setCredentials(scope, c); + return clientContext; } /** @@ -411,16 +413,22 @@ private void setRequestEntity(HttpRequestBody body) req.setEntity(entity); } + private enum State { + INITIAL, + POST_CONNECT + } + + private final HttpClientConnectionManager clientConnectionManager = new BasicHttpClientConnectionManager(); + private State state = State.INITIAL; + /** The target URI. */ private URI myUri; /** The Apache request. */ - private HttpUriRequest myRequest; + private HttpRequestBase myRequest; /** The Apache response. */ private HttpResponse myResponse; /** The HTTP protocol version. */ private HttpVersion myVersion; - /** The Apache client. */ - private AbstractHttpClient myClient; /** Follow HTTP redirect? */ private boolean myFollowRedirect = true; /** The timeout to use, in seconds, or null for default. */ diff --git a/http-client-java/src/test/java/tmp/tests/ApacheHttpClientTest.java b/http-client-java/src/test/java/tmp/tests/ApacheHttpClientTest.java index afe0a3c..6d0ef39 100644 --- a/http-client-java/src/test/java/tmp/tests/ApacheHttpClientTest.java +++ b/http-client-java/src/test/java/tmp/tests/ApacheHttpClientTest.java @@ -7,7 +7,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; -import java.net.ProxySelector; import java.net.URI; import java.net.URISyntaxException; import java.security.KeyManagementException; @@ -21,22 +20,21 @@ import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; -import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.params.ClientPNames; -import org.apache.http.conn.routing.HttpRoutePlanner; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.ssl.SSLSocketFactory; +import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.cookie.Cookie; import org.apache.http.entity.ContentProducer; import org.apache.http.entity.EntityTemplate; -import org.apache.http.impl.client.AbstractHttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.ProxySelectorRoutePlanner; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.SystemDefaultRoutePlanner; +import org.apache.http.ssl.SSLContextBuilder; import org.junit.Ignore; import org.junit.Test; +import javax.net.ssl.SSLContext; + /** * * @author Florent Georges - fgeorges.org @@ -45,14 +43,16 @@ public class ApacheHttpClientTest { @Test public void testGetMethod() - throws ClientProtocolException, IOException + throws IOException { HttpGet get = new HttpGet("http://www.fgeorges.org/"); - HttpResponse response = getClient().execute(get); - System.err.println("Status: " + response.getStatusLine().getStatusCode()); - System.err.print("Content: "); - response.getEntity().writeTo(System.err); - System.err.println(); + try(final CloseableHttpClient client = getClient()) { + HttpResponse response = client.execute(get); + System.err.println("Status: " + response.getStatusLine().getStatusCode()); + System.err.print("Content: "); + response.getEntity().writeTo(System.err); + System.err.println(); + } } /** @@ -60,70 +60,80 @@ public void testGetMethod() */ @Test public void testGoogleAuth() - throws ClientProtocolException, IOException + throws IOException { HttpPost post = new HttpPost("https://www.google.com/accounts/ClientLogin"); ContentProducer producer = new StringProducer(AUTH_CONTENT); EntityTemplate entity = new EntityTemplate(producer); entity.setContentType(FORM_TYPE); post.setEntity(entity); - HttpResponse resp = getClient().execute(post); - System.err.println("Status: " + resp.getStatusLine().getStatusCode()); - System.err.print("Content: "); - resp.getEntity().writeTo(System.err); - System.err.println(); + try(final CloseableHttpClient client = getClient()) { + HttpResponse resp = client.execute(post); + System.err.println("Status: " + resp.getStatusLine().getStatusCode()); + System.err.print("Content: "); + resp.getEntity().writeTo(System.err); + System.err.println(); + } } @Test public void testGoogleRedirect() - throws ClientProtocolException, IOException + throws IOException { HttpPost post = new HttpPost("https://www.google.com/accounts/ClientLogin"); ContentProducer producer = new StringProducer(AUTH_CONTENT); EntityTemplate entity = new EntityTemplate(producer); entity.setContentType(FORM_TYPE); post.setEntity(entity); - AbstractHttpClient client = getClient(); - HttpResponse resp = client.execute(post); - System.err.println("POST status: " + resp.getStatusLine().getStatusCode()); String token = null; - for ( String s : getStringContent(resp).split("\n") ) { - if ( s.startsWith("Auth=") ) { - token = s.substring(5); + try(final CloseableHttpClient client = getClient()) { + HttpResponse resp = client.execute(post); + System.err.println("POST status: " + resp.getStatusLine().getStatusCode()); + for (String s : getStringContent(resp).split("\n")) { + if (s.startsWith("Auth=")) { + token = s.substring(5); + } } + System.err.println("Token: " + token); } - System.err.println("Token: " + token); + // GetMethod get = new GetMethod("https://www.google.com/calendar/feeds/default/allcalendars/full"); HttpGet get = new HttpGet("http://www.google.com/calendar/feeds/xmlprague.cz_k0rlr8da52ivmgp6eujip041s8%40group.calendar.google.com/private/full"); get.setHeader("GData-Version", "2"); get.setHeader("Authorization", "GoogleLogin auth=" + token); // get.setFollowRedirects(false); - resp = client.execute(get); - System.err.println("GET status: " + resp.getStatusLine().getStatusCode()); - for ( Cookie c : client.getCookieStore().getCookies() ) { - System.err.println("Cookie: " + c.getName() + ", " + c.getValue()); - } - for ( String s : client.getCookieSpecs().getSpecNames() ) { - System.err.println("Cookie spec: " + s); - } - if ( resp.getStatusLine().getStatusCode() == 302 ) { - client = getClient(); - resp = client.execute(get); + HttpClientContext context = HttpClientContext.create(); + try(final CloseableHttpClient client = getClient()) { + HttpResponse resp = client.execute(get, context); System.err.println("GET status: " + resp.getStatusLine().getStatusCode()); + + for (Cookie c : context.getCookieStore().getCookies()) { + System.err.println("Cookie: " + c.getName() + ", " + c.getValue()); + } + System.err.println("Cookie spec: " + context.getCookieSpec()); + + if (resp.getStatusLine().getStatusCode() == 302) { + try(final CloseableHttpClient client2 = getClient()) { + final HttpResponse resp2 = client2.execute(get); + System.err.println("GET status: " + resp2.getStatusLine().getStatusCode()); + } + } } + HttpGet get2 = new HttpGet("http://www.google.com/calendar/feeds/xmlprague.cz_k0rlr8da52ivmgp6eujip041s8%40group.calendar.google.com/private/full"); get2.setHeader("GData-Version", "2"); get2.setHeader("Authorization", "GoogleLogin auth=" + token); // get2.setFollowRedirects(false); - client = getClient(); - resp = client.execute(get2); - System.err.println("GET 2 status: " + resp.getStatusLine().getStatusCode()); + try(final CloseableHttpClient client = getClient()) { + HttpResponse resp = client.execute(get2); + System.err.println("GET 2 status: " + resp.getStatusLine().getStatusCode()); + } } @Ignore("Broken authentication?!?") @Test public void testGoogleAddAgenda() - throws ClientProtocolException, IOException + throws IOException { System.err.println(); System.err.println("***** [testGoogleAddAgenda]"); @@ -137,8 +147,8 @@ public void testGoogleAddAgenda() System.err.println("***** [/testGoogleAddAgenda]"); } - public HttpResponse testGoogleAddAgenda_1(String token, String uri) - throws ClientProtocolException, IOException + private HttpResponse testGoogleAddAgenda_1(String token, String uri) + throws IOException { HttpPost post = new HttpPost(uri); post.setHeader("GData-Version", "2"); @@ -146,19 +156,21 @@ public HttpResponse testGoogleAddAgenda_1(String token, String uri) EntityTemplate entity = new EntityTemplate(new StringProducer(AGENDA_ENTRY)); entity.setContentType(ATOM_TYPE); post.setEntity(entity); - HttpResponse resp = getClient().execute(post); - System.err.println("POST status: " + resp.getStatusLine().getStatusCode()); - System.err.println("POST message: " + resp.getStatusLine().getReasonPhrase()); - System.err.print("POST response content: "); - resp.getEntity().writeTo(System.err); - System.err.println(); - return resp; + try(final CloseableHttpClient client = getClient()) { + HttpResponse resp = client.execute(post); + System.err.println("POST status: " + resp.getStatusLine().getStatusCode()); + System.err.println("POST message: " + resp.getStatusLine().getReasonPhrase()); + System.err.print("POST response content: "); + resp.getEntity().writeTo(System.err); + System.err.println(); + return resp; + } } @Ignore("Broken authentication?!?") @Test public void testGoogleAddAgendaStd() - throws ClientProtocolException, IOException, URISyntaxException + throws IOException, URISyntaxException { System.err.println(); System.err.println("***** [testGoogleAddAgendaStd]"); @@ -174,7 +186,7 @@ public void testGoogleAddAgendaStd() } private HttpURLConnection testGoogleAddAgendaStd_1(String token, URI uri) - throws ClientProtocolException, IOException, URISyntaxException + throws IOException, URISyntaxException { HttpURLConnection conn = (HttpURLConnection) uri.toURL().openConnection(); conn.setRequestMethod("POST"); @@ -200,44 +212,48 @@ private String authenticate() EntityTemplate entity = new EntityTemplate(new StringProducer(AUTH_CONTENT)); entity.setContentType(FORM_TYPE); auth.setEntity(entity); - HttpResponse resp = getClient().execute(auth); - System.err.println("AUTH status: " + resp.getStatusLine().getStatusCode()); - String token = null; - for ( String s : getStringContent(resp).split("\n") ) { - System.err.println("AUTH content line: " + s); - if ( s.startsWith("Auth=") ) { - token = s.substring(5); + try(final CloseableHttpClient client = getClient()) { + HttpResponse resp = client.execute(auth); + System.err.println("AUTH status: " + resp.getStatusLine().getStatusCode()); + String token = null; + for (String s : getStringContent(resp).split("\n")) { + System.err.println("AUTH content line: " + s); + if (s.startsWith("Auth=")) { + token = s.substring(5); + } } + if (token == null) { + throw new RuntimeException("Token is null!"); + } + return token; } - if ( token == null ) { - throw new RuntimeException("Token is null!"); - } - return token; } @Test public void testResponseBody() - throws ClientProtocolException, IOException, URISyntaxException + throws IOException, URISyntaxException { System.err.println(); System.err.println("***** [testResponseBody]"); HttpGet get = new HttpGet("http://www.fgeorges.org/tmp/xproc-fixed-alternative.mpr-"); - HttpResponse resp = getClient().execute(get); - System.err.println("Status: " + resp.getStatusLine().getStatusCode()); - HttpEntity entity = resp.getEntity(); - System.err.println("Entity class: " + entity.getClass()); - System.err.println("Entity type: " + entity.getContentType()); - System.err.println("Entity encoding: " + entity.getContentEncoding()); - System.err.println("Entity is chunck: " + entity.isChunked()); - System.err.println("Entity is repeat: " + entity.isRepeatable()); - System.err.println("Entity is stream: " + entity.isStreaming()); - System.err.println("***** [/testResponseBody]"); + try(final CloseableHttpClient client = getClient()) { + HttpResponse resp = client.execute(get); + System.err.println("Status: " + resp.getStatusLine().getStatusCode()); + HttpEntity entity = resp.getEntity(); + System.err.println("Entity class: " + entity.getClass()); + System.err.println("Entity type: " + entity.getContentType()); + System.err.println("Entity encoding: " + entity.getContentEncoding()); + System.err.println("Entity is chunck: " + entity.isChunked()); + System.err.println("Entity is repeat: " + entity.isRepeatable()); + System.err.println("Entity is stream: " + entity.isStreaming()); + System.err.println("***** [/testResponseBody]"); + } } @Ignore("Need a certificate file, see 'Unable to find the certificate file' below") @Test public void testTrustSelfSignedKeys() - throws ClientProtocolException, IOException, URISyntaxException, + throws IOException, URISyntaxException, KeyStoreException, NoSuchAlgorithmException, CertificateException, KeyManagementException, UnrecoverableKeyException { @@ -255,24 +271,28 @@ public void testTrustSelfSignedKeys() InputStream in = new FileInputStream(in_f); Certificate certif = factory.generateCertificate(in); trustStore.setCertificateEntry("fgeorges.org", certif); - SSLSocketFactory socketFactory = new SSLSocketFactory(trustStore); - Scheme sch = new Scheme("https", socketFactory, 443); - AbstractHttpClient client = getClient(); - client.getConnectionManager().getSchemeRegistry().register(sch); - // - System.err.println(); - System.err.println("***** [testTrustSelfSignedKeys]"); - HttpGet get = new HttpGet("https://www.fgeorges.org/"); - //HttpGet get = new HttpGet("https://mail.google.com/"); - HttpResponse resp = client.execute(get); - System.err.println("Status: " + resp.getStatusLine().getStatusCode()); - System.err.println("***** [/testTrustSelfSignedKeys]"); + final SSLContext sslContext = SSLContextBuilder.create() + .loadTrustMaterial(trustStore, null) + .build(); + + try (CloseableHttpClient client = getClient(sslContext)) { + + // + + System.err.println(); + System.err.println("***** [testTrustSelfSignedKeys]"); + HttpGet get = new HttpGet("https://www.fgeorges.org/"); + //HttpGet get = new HttpGet("https://mail.google.com/"); + HttpResponse resp = client.execute(get); + System.err.println("Status: " + resp.getStatusLine().getStatusCode()); + System.err.println("***** [/testTrustSelfSignedKeys]"); + } } @Test public void testXProcPost() - throws ClientProtocolException, IOException, URISyntaxException + throws IOException, URISyntaxException { System.err.println(); System.err.println("***** [testXProcPost]"); @@ -284,7 +304,7 @@ public void testXProcPost() @Test public void testFGeorgesPost() - throws ClientProtocolException, IOException, URISyntaxException + throws IOException, URISyntaxException { System.err.println(); System.err.println("***** [testFGeorgesPost]"); @@ -295,31 +315,32 @@ public void testFGeorgesPost() } private void doPost(String uri, String content, String type) - throws ClientProtocolException, IOException, URISyntaxException + throws IOException, URISyntaxException { HttpPost post = new HttpPost(uri); EntityTemplate entity = new EntityTemplate(new StringProducer(content)); entity.setContentType(type); post.setEntity(entity); - AbstractHttpClient client = getClient(); - System.err.println("DEBUG: CLIENT: " + client.getClass()); - HttpResponse resp = client.execute(post); - System.err.println("Status: " + resp.getStatusLine().getStatusCode()); - HttpEntity body = resp.getEntity(); - System.err.println("Entity class: " + body.getClass()); - System.err.println("Entity type: " + body.getContentType()); - System.err.println("Entity encoding: " + body.getContentEncoding()); - System.err.println("Entity is chunck: " + body.isChunked()); - System.err.println("Entity is repeat: " + body.isRepeatable()); - System.err.println("Entity is stream: " + body.isStreaming()); - System.err.println("Entity body: "); - body.writeTo(System.err); - System.err.println(); + try(CloseableHttpClient client = getClient()) { + System.err.println("DEBUG: CLIENT: " + client.getClass()); + HttpResponse resp = client.execute(post); + System.err.println("Status: " + resp.getStatusLine().getStatusCode()); + HttpEntity body = resp.getEntity(); + System.err.println("Entity class: " + body.getClass()); + System.err.println("Entity type: " + body.getContentType()); + System.err.println("Entity encoding: " + body.getContentEncoding()); + System.err.println("Entity is chunck: " + body.isChunked()); + System.err.println("Entity is repeat: " + body.isRepeatable()); + System.err.println("Entity is stream: " + body.isStreaming()); + System.err.println("Entity body: "); + body.writeTo(System.err); + System.err.println(); + } } @Test public void testPost() - throws ClientProtocolException, IOException, URISyntaxException + throws IOException, URISyntaxException { System.err.println(); System.err.println("***** [testPost]"); @@ -328,10 +349,12 @@ public void testPost() EntityTemplate entity = new EntityTemplate(new StringProducer("")); entity.setContentType(XML_TYPE); post.setEntity(entity); - HttpResponse resp = getClient().execute(post); - System.err.println("Status: " + resp.getStatusLine().getStatusCode()); - System.err.println(); - System.err.println("***** [/testPost]"); + try(CloseableHttpClient client = getClient()) { + HttpResponse resp = client.execute(post); + System.err.println("Status: " + resp.getStatusLine().getStatusCode()); + System.err.println(); + System.err.println("***** [/testPost]"); + } } private String getStringContent(HttpResponse resp) @@ -364,23 +387,30 @@ public void writeTo(OutputStream out) throws IOException { private byte[] myContent; } - private static AbstractHttpClient getClient() + private static CloseableHttpClient getClient() + { + // FIXME: TODO: How to manage and reuse connections? In test cases, but + // also in production code... + return getClient(null); +// return CLIENT; + } + + private static CloseableHttpClient getClient(final SSLContext sslContext) { // FIXME: TODO: How to manage and reuse connections? In test cases, but // also in production code... - return makeNewClient(); + return makeNewClient(sslContext); // return CLIENT; } - private static AbstractHttpClient makeNewClient() + private static CloseableHttpClient makeNewClient(final SSLContext sslContext) { - AbstractHttpClient client = new DefaultHttpClient(); - HttpRoutePlanner routePlanner = new ProxySelectorRoutePlanner( - client.getConnectionManager().getSchemeRegistry(), - ProxySelector.getDefault()); - client.setRoutePlanner(routePlanner); - client.getParams().setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, false); - return client; + final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); + if(sslContext != null) { + clientBuilder.setSSLContext(sslContext); + } + clientBuilder.setRoutePlanner(new SystemDefaultRoutePlanner(null)); + return clientBuilder.build(); } static { @@ -397,7 +427,7 @@ private static AbstractHttpClient makeNewClient() private static final String FORM_TYPE = "application/x-www-form-urlencoded"; private static final String ATOM_TYPE = "application/atom+xml"; private static final String XML_TYPE = "application/xml"; - private static final AbstractHttpClient CLIENT = makeNewClient(); + private static final CloseableHttpClient CLIENT = makeNewClient(null); // private static final AbstractHttpClient CLIENT = new DefaultHttpClient(); // static { // HttpRoutePlanner routePlanner = new ProxySelectorRoutePlanner( From dd3c441c948d101376445c776e13a1a042ca7532 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 17 Aug 2017 16:34:57 -0400 Subject: [PATCH 11/27] [feature] Implemented connection pooling --- .../httpclient/impl/ApacheHttpConnection.java | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 6b58c67..76bf321 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -11,13 +11,13 @@ import java.io.*; import java.net.URI; +import java.util.concurrent.TimeUnit; import net.jcip.annotations.NotThreadSafe; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.Header; import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; import org.apache.http.HttpVersion; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; @@ -26,7 +26,6 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.*; import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentProducer; import org.apache.http.entity.EntityTemplate; @@ -56,7 +55,8 @@ public ApacheHttpConnection(URI uri) myVersion = DEFAULT_HTTP_VERSION; } - public void connect(HttpRequestBody body, HttpCredentials cred) + @Override + public void connect(final HttpRequestBody body, final HttpCredentials cred) throws HttpClientException { if ( myRequest == null ) { @@ -67,7 +67,15 @@ public void connect(HttpRequestBody body, HttpCredentials cred) try { // make a new client - final CloseableHttpClient myClient = makeClient(); + if(myClient == null) { + myClient = makeClient(); + } + + if(myResponse != null) { + // close any previous response + myResponse.close(); + } + // set the credentials (if any) final HttpClientContext clientContext = setCredentials(cred); // set the request entity body (if any) @@ -124,8 +132,19 @@ private String getMessage(final Throwable throwable) { } @Override - public void disconnect() { - clientConnectionManager.shutdown(); + public void disconnect() throws HttpClientException { + try { + if(myResponse != null) { + myResponse.close(); + myResponse = null; + } + + myClient.close(); + myClient = null; + } catch (final IOException ex) { + final String message = getMessage(ex); + throw new HttpClientException(message, ex); + } } @Override @@ -299,8 +318,9 @@ public InputStream getResponseStream() */ private CloseableHttpClient makeClient() { - final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); - clientBuilder.setConnectionManager(clientConnectionManager); + final HttpClientBuilder clientBuilder = HttpClientBuilder.create() + .setConnectionManager(POOLING_CONNECTION_MANAGER) + .setConnectionManagerShared(true); // use the default JVM proxy settings (http.proxyHost, etc.) clientBuilder.setRoutePlanner(new SystemDefaultRoutePlanner(null)); @@ -418,15 +438,21 @@ private enum State { POST_CONNECT } - private final HttpClientConnectionManager clientConnectionManager = new BasicHttpClientConnectionManager(); + private static final PoolingHttpClientConnectionManager POOLING_CONNECTION_MANAGER + = new PoolingHttpClientConnectionManager(15, TimeUnit.MINUTES); //TODO(AR) make configurable + static { + POOLING_CONNECTION_MANAGER.setMaxTotal(40); //TODO(AR) make configurable + } private State state = State.INITIAL; /** The target URI. */ private URI myUri; + /** The Apache client. */ + private CloseableHttpClient myClient; /** The Apache request. */ private HttpRequestBase myRequest; /** The Apache response. */ - private HttpResponse myResponse; + private CloseableHttpResponse myResponse; /** The HTTP protocol version. */ private HttpVersion myVersion; /** Follow HTTP redirect? */ From 80d5cdce4a2c7c724b116d230c47526de77332d5 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 17 Aug 2017 17:01:55 -0400 Subject: [PATCH 12/27] [feature] Support for transferring GZipped data, just add a gzip="true" attribute to the http:request element --- .../org/expath/httpclient/HttpConnection.java | 1 + .../org/expath/httpclient/HttpRequest.java | 1 + .../httpclient/impl/ApacheHttpConnection.java | 27 +++++++++++++++++-- .../httpclient/impl/HttpRequestImpl.java | 10 +++++++ .../expath/httpclient/impl/RequestParser.java | 3 +++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java index 1e662e1..cadf8dc 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java @@ -34,6 +34,7 @@ public void setRequestMethod(String method, boolean with_content) throws HttpClientException; public void setFollowRedirect(boolean follow); public void setTimeout(int seconds); + public void setGzip(boolean gzip); // responses... public int getResponseStatus() throws HttpClientException; diff --git a/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java index 3c25686..7a46c9b 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java @@ -34,6 +34,7 @@ public void setBody(HttpRequestBody body) public void setStatusOnly(boolean only); public void setFollowRedirect(boolean follow); public void setTimeout(Integer seconds); + public void setGzip(boolean gzip); } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 76bf321..3a32703 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -12,6 +12,7 @@ import java.io.*; import java.net.URI; import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPOutputStream; import net.jcip.annotations.NotThreadSafe; import org.apache.commons.logging.Log; @@ -24,6 +25,7 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CookieStore; import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.GzipCompressingEntity; import org.apache.http.client.methods.*; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.entity.ByteArrayEntity; @@ -31,6 +33,7 @@ import org.apache.http.entity.EntityTemplate; import org.apache.http.impl.client.*; import org.apache.http.impl.conn.*; +import org.apache.http.protocol.HTTP; import org.expath.httpclient.HeaderSet; import org.expath.httpclient.HttpClientException; import org.expath.httpclient.HttpConnection; @@ -226,6 +229,11 @@ public void setTimeout(int seconds) myTimeout = seconds; } + @Override + public void setGzip(final boolean gzip) { + myGzip = gzip; + } + /** * Check the method name does match the HTTP/1.1 production rules. * @@ -406,14 +414,26 @@ private void setRequestEntity(HttpRequestBody body) EntityTemplate template = new EntityTemplate(producer); template.setContentType(body.getContentType()); template.setChunked(true); - entity = template; + + if(myGzip) { + entity = new GzipCompressingEntity(template); + } else { + entity = template; + } } else { // With HTTP 1.0, chunked encoding is not supported, so first // serialize into memory and use the resulting byte array as the // entity payload. try (final ByteArrayOutputStream buffer = new ByteArrayOutputStream()) { - body.serialize(buffer); + if(myGzip) { + try (final GZIPOutputStream gzip = new GZIPOutputStream(buffer)) { + body.serialize(gzip); + } + myRequest.setHeader(HTTP.CONTENT_ENCODING, "gzip"); + } else { + body.serialize(buffer); + } entity = new ByteArrayEntity(buffer.toByteArray()); } catch (final IOException e) { throw new HttpClientException(e.getMessage(), e); @@ -459,6 +479,9 @@ private enum State { private boolean myFollowRedirect = true; /** The timeout to use, in seconds, or null for default. */ private Integer myTimeout = null; + /** whether we should use gzip transfer encoding */ + private boolean myGzip = false; + /** * The shared cookie store. * diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java index ba5e1b4..1dcec5d 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java @@ -47,6 +47,10 @@ public HttpResponse send(Result result, HttpConnection conn, HttpCredentials cre if ( myTimeout != null ) { conn.setTimeout(myTimeout); } + if ( myGzip ) { + conn.setGzip(true); + } + conn.setFollowRedirect(myFollowRedirect); conn.connect(myBody, cred); int status = conn.getResponseStatus(); @@ -169,6 +173,11 @@ public void setTimeout(Integer seconds) myTimeout = seconds; } + @Override + public void setGzip(final boolean gzip) { + myGzip = gzip; + } + private String myMethod; private String myHref; private String myHttpVer; @@ -176,6 +185,7 @@ public void setTimeout(Integer seconds) private boolean myStatusOnly; private boolean myFollowRedirect = true; private Integer myTimeout = null; + private boolean myGzip = false; private HeaderSet myHeaders; private HttpRequestBody myBody; private static final Log LOG = LogFactory.getLog(HttpRequestImpl.class); diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java index babb5ce..d1b3b6c 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java @@ -127,6 +127,9 @@ else if ( "follow-redirect".equals(local) ) { else if ( "timeout".equals(local) ) { req.setTimeout(toInteger(a)); } + else if ( "gzip".equals(local) ) { + req.setGzip(toBoolean(a)); + } else { throw new HttpClientException("Unknown attribute http:request/@" + local); } From 27a0640bcc6fa7f280d10ff715a80fa6a53d8049 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Mon, 21 Aug 2017 10:49:33 -0400 Subject: [PATCH 13/27] [feature] Added support for SNI (Server Name Identification) Closes https://github.com/fgeorges/expath-http-client-java/issues/5 --- .../httpclient/impl/ApacheHttpConnection.java | 67 +++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 3a32703..44ab120 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -10,6 +10,10 @@ package org.expath.httpclient.impl; import java.io.*; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.InetSocketAddress; +import java.net.Socket; import java.net.URI; import java.util.concurrent.TimeUnit; import java.util.zip.GZIPOutputStream; @@ -19,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; import org.apache.http.HttpVersion; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; @@ -28,12 +33,19 @@ import org.apache.http.client.entity.GzipCompressingEntity; import org.apache.http.client.methods.*; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentProducer; import org.apache.http.entity.EntityTemplate; import org.apache.http.impl.client.*; import org.apache.http.impl.conn.*; import org.apache.http.protocol.HTTP; +import org.apache.http.protocol.HttpContext; +import org.apache.http.ssl.SSLContexts; import org.expath.httpclient.HeaderSet; import org.expath.httpclient.HttpClientException; import org.expath.httpclient.HttpConnection; @@ -41,6 +53,9 @@ import org.expath.httpclient.HttpCredentials; import org.expath.httpclient.HttpRequestBody; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; + /** * An implementation of an HTTP connection using Apachhe HTTP Client. * @@ -453,16 +468,58 @@ private void setRequestEntity(HttpRequestBody body) req.setEntity(entity); } + private static PoolingHttpClientConnectionManager setupConnectionPool() { + final SSLContext sslContext = SSLContexts. + createSystemDefault(); + + final SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLSocketFactoryWithSNI(sslContext); + + final Registry socketFactoryRegistry = RegistryBuilder.create() + .register("https", sslConnectionSocketFactory) + .register("http", PlainConnectionSocketFactory.INSTANCE) + .build(); + + final PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, 15, TimeUnit.MINUTES); //TODO(AR) TTL is currently 15 minutes, make configurable? + poolingHttpClientConnectionManager.setMaxTotal(40); //TODO(AR) total pooled connections is 40, make configurable? + return poolingHttpClientConnectionManager; + } + + /** + * Implements SNI (Server Name Identification) for SSL + * + * @see https://github.com/fgeorges/expath-http-client-java/issues/5 + */ + private static class SSLSocketFactoryWithSNI extends SSLConnectionSocketFactory { + public SSLSocketFactoryWithSNI(final SSLContext sslContext) { + super(sslContext); + } + + @Override + public Socket connectSocket(final int connectTimeout, final Socket socket, final HttpHost host, + final InetSocketAddress remoteAddress, final InetSocketAddress localAddress, final HttpContext context) + throws IOException { + if (socket instanceof SSLSocket) { + try { + final Class socketClazz = socket.getClass(); + final Method m = socketClazz.getDeclaredMethod("setHost", String.class); + m.setAccessible(true); + m.invoke(socket, host.getHostName()); + } catch (final NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + LOG.warn("Problem whilst setting SNI: " + e.getMessage(), e); + } + } + + return super.connectSocket(connectTimeout, socket, host, remoteAddress, localAddress, context); + } + } + private enum State { INITIAL, POST_CONNECT } - private static final PoolingHttpClientConnectionManager POOLING_CONNECTION_MANAGER - = new PoolingHttpClientConnectionManager(15, TimeUnit.MINUTES); //TODO(AR) make configurable - static { - POOLING_CONNECTION_MANAGER.setMaxTotal(40); //TODO(AR) make configurable - } + private static final PoolingHttpClientConnectionManager POOLING_CONNECTION_MANAGER = setupConnectionPool(); + private State state = State.INITIAL; /** The target URI. */ From f5388a5a8e27352f628a33dc8b43068e9da9e8d9 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Mon, 28 Aug 2017 13:33:39 -0400 Subject: [PATCH 14/27] [bugfix] Fix an issue in DOM namespace handling --- .../main/java/org/expath/httpclient/impl/RequestParser.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java index d1b3b6c..061cff7 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java @@ -91,7 +91,7 @@ public HttpRequest parse(Sequence bodies, String href) // timeout? = integer for ( Attribute a : myRequest.attributes() ) { String local = a.getLocalName(); - if ( !"".equals(a.getNamespaceUri()) ) { + if ( !(a.getNamespaceUri() == null || a.getNamespaceUri().isEmpty()) ) { // ignore namespace qualified attributes } else if ( "method".equals(local) ) { @@ -151,7 +151,7 @@ else if ( "gzip".equals(local) ) { for ( Element child : myRequest.children() ) { String local = child.getLocalName(); String ns = child.getNamespaceUri(); - if ( "".equals(ns) ) { + if ( ns == null || ns.isEmpty() ) { // elements in no namespace are an error throw new HttpClientException("Element in no namespace: " + local); } @@ -203,7 +203,7 @@ private void addHeader(HeaderSet headers, Element e) String value = null; for ( Attribute a : e.attributes() ) { String local = a.getLocalName(); - if ( !"".equals(a.getNamespaceUri()) ) { + if ( !(a.getNamespaceUri() == null || a.getNamespaceUri().isEmpty()) ) { // ignore namespace qualified attributes } else if ( "name".equals(local) ) { From 8b03e48e1d2bce7ac6e4f4c53c388279f9d70555 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Mon, 25 Sep 2017 18:05:11 +0100 Subject: [PATCH 15/27] [feature] Add the ability to do non-chunked transfer encoding for HTTP 1.1. Chunked remains the default for HTTP 1.1 --- .../org/expath/httpclient/HttpConnection.java | 1 + .../org/expath/httpclient/HttpRequest.java | 2 ++ .../httpclient/impl/ApacheHttpConnection.java | 36 +++++++++++++++---- .../httpclient/impl/HttpRequestImpl.java | 20 +++++++++++ .../expath/httpclient/impl/RequestParser.java | 8 +++++ 5 files changed, 61 insertions(+), 6 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java index cadf8dc..7ce5cea 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java @@ -35,6 +35,7 @@ public void setRequestMethod(String method, boolean with_content) public void setFollowRedirect(boolean follow); public void setTimeout(int seconds); public void setGzip(boolean gzip); + public void setChunked(boolean chunked); // responses... public int getResponseStatus() throws HttpClientException; diff --git a/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java index 7a46c9b..c5da7ce 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java @@ -35,6 +35,8 @@ public void setBody(HttpRequestBody body) public void setFollowRedirect(boolean follow); public void setTimeout(Integer seconds); public void setGzip(boolean gzip); + public boolean isChunked(); + public void setChunked(boolean chunked); } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 44ab120..3789578 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -249,6 +249,11 @@ public void setGzip(final boolean gzip) { myGzip = gzip; } + @Override + public void setChunked(final boolean chunked) { + myChunked = chunked; + } + /** * Check the method name does match the HTTP/1.1 production rules. * @@ -423,12 +428,30 @@ private void setRequestEntity(HttpRequestBody body) // make the entity from a new producer final HttpEntity entity; if ( myVersion == HttpVersion.HTTP_1_1 ) { - // Take advantage of HTTP 1.1 chunked encoding to stream the - // payload directly to the request. - ContentProducer producer = new RequestBodyProducer(body); - EntityTemplate template = new EntityTemplate(producer); - template.setContentType(body.getContentType()); - template.setChunked(true); + + final HttpEntity template; + if(myChunked) { + // Take advantage of HTTP 1.1 chunked encoding to stream the + // payload directly to the request. + final ContentProducer producer = new RequestBodyProducer(body); + final EntityTemplate entityTemplate = new EntityTemplate(producer); + entityTemplate.setContentType(body.getContentType()); + entityTemplate.setChunked(true); + template = entityTemplate; + + } else { + /* + NOTE: for some reason even if you set EntityTemplate#setChunked(false), + Apache insists on chunking anyway... So, instead we manually buffer here + to foce non-chunked transfer encoding. + */ + try (final ByteArrayOutputStream buffer = new ByteArrayOutputStream()) { + body.serialize(buffer); + template = new ByteArrayEntity(buffer.toByteArray()); + } catch (final IOException e) { + throw new HttpClientException(e.getMessage(), e); + } + } if(myGzip) { entity = new GzipCompressingEntity(template); @@ -538,6 +561,7 @@ private enum State { private Integer myTimeout = null; /** whether we should use gzip transfer encoding */ private boolean myGzip = false; + private boolean myChunked = true; /** * The shared cookie store. diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java index 1dcec5d..635808a 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java @@ -50,6 +50,7 @@ public HttpResponse send(Result result, HttpConnection conn, HttpCredentials cre if ( myGzip ) { conn.setGzip(true); } + conn.setChunked(isChunked()); conn.setFollowRedirect(myFollowRedirect); conn.connect(myBody, cred); @@ -178,6 +179,24 @@ public void setGzip(final boolean gzip) { myGzip = gzip; } + @Override + public boolean isChunked() { + if(myChunked == null) { + if(myHttpVer.equals(HttpConstants.HTTP_1_0)) { + return false; + } else { + return true; + } + } else { + return myChunked; + } + } + + @Override + public void setChunked(boolean chunked) { + this.myChunked = chunked; + } + private String myMethod; private String myHref; private String myHttpVer; @@ -186,6 +205,7 @@ public void setGzip(final boolean gzip) { private boolean myFollowRedirect = true; private Integer myTimeout = null; private boolean myGzip = false; + private Boolean myChunked = null; private HeaderSet myHeaders; private HttpRequestBody myBody; private static final Log LOG = LogFactory.getLog(HttpRequestImpl.class); diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java index 061cff7..c6117aa 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java @@ -89,6 +89,8 @@ public HttpRequest parse(Sequence bodies, String href) // override-media-type? = string // follow-redirect? = boolean // timeout? = integer + // gzip? = boolean + // chunked? = boolean for ( Attribute a : myRequest.attributes() ) { String local = a.getLocalName(); if ( !(a.getNamespaceUri() == null || a.getNamespaceUri().isEmpty()) ) { @@ -130,6 +132,9 @@ else if ( "timeout".equals(local) ) { else if ( "gzip".equals(local) ) { req.setGzip(toBoolean(a)); } + else if ( "chunked".equals(local) ) { + req.setChunked(toBoolean(a)); + } else { throw new HttpClientException("Unknown attribute http:request/@" + local); } @@ -143,6 +148,9 @@ else if ( "gzip".equals(local) ) { if ( username != null || password != null || auth_method != null ) { setAuthentication(username, password, auth_method); } + if(req.getHttpVersion() != null && req.getHttpVersion().equals(HttpConstants.HTTP_1_0) && req.isChunked()) { + throw new HttpClientException("Chunked transfer encoding can only be used with HTTP 1.1"); + } // walk the elements // TODO: Check element structure validity (header*, (multipart|body)?) From 90d00ff8a4195aca6c4c011f348033e8a28be335 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Mon, 25 Sep 2017 18:07:43 +0100 Subject: [PATCH 16/27] [ignore] Ignore IntelliJ project files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a961cfc..b7cb0f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +*.iml +.idea/ + /build/expath-http-client-java-*/ /build/expath-http-client-saxon-*/ /build/expath-http-client-java-*.zip From 080869393ae8889fda79752e2592bd8adaa85801 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sat, 30 Sep 2017 11:44:52 +0100 Subject: [PATCH 17/27] [bugfix] Avoid NPE when doing non-chunked transfer encoding without specifiying HTTP version --- .../main/java/org/expath/httpclient/impl/HttpRequestImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java index 635808a..3e8847d 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java @@ -182,7 +182,7 @@ public void setGzip(final boolean gzip) { @Override public boolean isChunked() { if(myChunked == null) { - if(myHttpVer.equals(HttpConstants.HTTP_1_0)) { + if(myHttpVer != null && myHttpVer.equals(HttpConstants.HTTP_1_0)) { return false; } else { return true; From de19d7340e9b4945bbd2a7eda38c65907ec48cc1 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 16 Nov 2017 21:14:05 +0000 Subject: [PATCH 18/27] [bugfix] Fix NPE when using Basic Authentication --- .../org/expath/httpclient/impl/ApacheHttpConnection.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 3789578..915bb8a 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -412,6 +412,12 @@ else if ( "https".equals(scheme) ) { final Credentials c = new UsernamePasswordCredentials(user, pwd); final AuthScope scope = new AuthScope(host, port); + if(clientContext.getCredentialsProvider() == null) { + clientContext.setCredentialsProvider(new BasicCredentialsProvider()); + } else { + clientContext.getCredentialsProvider().clear(); + } + clientContext.getCredentialsProvider().setCredentials(scope, c); return clientContext; } From a6d63c6c942b6f6bc1f6bb1a1659cffbbad07416 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 19 Jan 2018 14:28:00 +0000 Subject: [PATCH 19/27] [feature] Update to the latest Apache HTTP Components --- http-client-parent/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http-client-parent/pom.xml b/http-client-parent/pom.xml index 496d4ad..61f7ef7 100644 --- a/http-client-parent/pom.xml +++ b/http-client-parent/pom.xml @@ -48,12 +48,12 @@ org.apache.httpcomponents httpcore - 4.4.6 + 4.4.9 org.apache.httpcomponents httpclient - 4.5.3 + 4.5.4 From e9ac210d7a828e3d0b87235bb7a8ad520859bfea Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 19 Jan 2018 14:28:13 +0000 Subject: [PATCH 20/27] [ignore] Added a comment about config options --- .../java/org/expath/httpclient/impl/ApacheHttpConnection.java | 1 + 1 file changed, 1 insertion(+) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 915bb8a..04440b9 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -510,6 +510,7 @@ private static PoolingHttpClientConnectionManager setupConnectionPool() { final PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, 15, TimeUnit.MINUTES); //TODO(AR) TTL is currently 15 minutes, make configurable? poolingHttpClientConnectionManager.setMaxTotal(40); //TODO(AR) total pooled connections is 40, make configurable? + poolingHttpClientConnectionManager.setDefaultMaxPerRoute(2); //TODO(AR) max default connections per route is 2, make configurable? return poolingHttpClientConnectionManager; } From 03592e8887838b2f383ab10becb24ca8f800b551 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 5 Apr 2018 11:06:36 +0100 Subject: [PATCH 21/27] [feature/bugfix] Implement missing response charset encoding --- http-client-java/pom.xml | 4 + .../org/expath/httpclient/ContentType.java | 107 +++++++---- .../java/org/expath/httpclient/HeaderSet.java | 33 ---- .../expath/httpclient/impl/BodyFactory.java | 173 ++++++++---------- .../httpclient/impl/HttpRequestImpl.java | 2 +- .../impl/MultipartResponseBody.java | 126 ++++++------- .../httpclient/impl/TextResponseBody.java | 59 +++--- .../httpclient/impl/XmlResponseBody.java | 51 +++--- .../org/expath/httpclient/model/Result.java | 4 +- .../expath/httpclient/saxon/SaxonResult.java | 3 +- 10 files changed, 276 insertions(+), 286 deletions(-) diff --git a/http-client-java/pom.xml b/http-client-java/pom.xml index 7909d5b..4b6e971 100644 --- a/http-client-java/pom.xml +++ b/http-client-java/pom.xml @@ -79,6 +79,10 @@ net.jcip jcip-annotations + + com.google.code.findbugs + jsr305 + junit junit diff --git a/http-client-java/src/main/java/org/expath/httpclient/ContentType.java b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java index df9e0de..0c6d71c 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/ContentType.java +++ b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java @@ -13,70 +13,101 @@ import org.apache.http.HeaderElement; import org.apache.http.NameValuePair; +import javax.annotation.Nullable; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + /** * Represent a Content-Type header. - * + *

* Provide the ability to get the boundary param in case of a multipart * content type on the one hand, and the ability to get only the MIME type * string without any param on the other hand. * * @author Florent Georges */ -public class ContentType -{ - public ContentType(String type, String boundary) - { +public class ContentType { + + public static final Charset DEFAULT_HTTP_CHARSET = StandardCharsets.ISO_8859_1; + + public ContentType(final String type, final String charset, final String boundary) { myHeader = null; myType = type; + myCharset = charset; myBoundary = boundary; } - public ContentType(Header h) - throws HttpClientException - { - if ( h == null ) { + public ContentType(final Header h) throws HttpClientException { + if (h == null) { throw new HttpClientException("Header is null"); } - if ( ! "Content-Type".equalsIgnoreCase(h.getName()) ) { + if (!"Content-Type".equalsIgnoreCase(h.getName())) { throw new HttpClientException("Header is not content type"); } - myHeader = h; - myType = HeaderSet.getHeaderWithoutParam(myHeader); - HeaderElement[] elems = h.getElements(); - if ( elems != null ) { - for ( HeaderElement e : elems ) { - for ( NameValuePair p : e.getParameters() ) { - if ( "boundary".equals(p.getName()) ) { - myBoundary = p.getValue(); - } + + this.myHeader = h; + + final HeaderElement[] elems = myHeader.getElements(); + if (elems == null || elems.length == 0) { + this.myType = null; + } else if (elems.length > 1) { + throw new HttpClientException("Multiple Content-Type headers"); + } else { + this.myType = elems[0].getName(); + } + + String charset = null; + String boundary = null; + if (elems != null) { + for (final HeaderElement e : elems) { + final NameValuePair nvpCharset = e.getParameterByName("charset"); + if (nvpCharset != null) { + charset = nvpCharset.getValue(); + } + final NameValuePair nvpBoundary = e.getParameterByName("boundary"); + if (nvpBoundary != null) { + boundary = nvpBoundary.getValue(); } } } + this.myCharset = charset; + this.myBoundary = boundary; } @Override - public String toString() - { - if ( myHeader == null ) { - return "Content-Type: " + getValue(); - } - else { + public String toString() { + if (myHeader == null) { + final StringBuilder builder = new StringBuilder("Content-Type: ").append(getValue()); + if (myCharset != null) { + builder.append("; charset=").append(myCharset); + } + if (myBoundary != null) { + builder.append("; boundary=").append(myBoundary); + } + + return builder.toString(); + } else { return myHeader.toString(); } } - public String getType() - { + @Nullable + public String getType() { return myType; } - public String getBoundary() - { + @Nullable + public String getCharset() { + return myCharset; + } + + @Nullable + public String getBoundary() { return myBoundary; } - public String getValue() - { + @Nullable + public String getValue() { // TODO: Why did I add the boundary before...? // if ( myHeader == null ) { // StringBuilder b = new StringBuilder(); @@ -89,20 +120,20 @@ public String getValue() // } // return b.toString(); // } - if ( myType != null ) { + if (myType != null) { return myType; } - if ( myHeader != null ) { + if (myHeader != null) { return myHeader.getValue(); - } - else { + } else { return null; } } - private Header myHeader; - private String myType; - private String myBoundary; + private final Header myHeader; + private final String myType; + private final String myCharset; + private final String myBoundary; } diff --git a/http-client-java/src/main/java/org/expath/httpclient/HeaderSet.java b/http-client-java/src/main/java/org/expath/httpclient/HeaderSet.java index 5354cfc..8fb4933 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HeaderSet.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HeaderSet.java @@ -101,39 +101,6 @@ public Header getFirstHeader(String name) return null; } - public String getFirstHeaderWithoutParam(String name) - throws HttpClientException - { - Header h = getFirstHeader(name); - return getHeaderWithoutParam(h); - } - - public static String getValueWithoutParam(String header_value) - throws HttpClientException - { - Header h = new BasicHeader("X-Dummy", header_value); - return getHeaderWithoutParam(h); - } - - public static String getHeaderWithoutParam(Header header) - throws HttpClientException - { - // get the content type, only the mime string, like "type/subtype" - if ( header != null ) { - HeaderElement[] elems = header.getElements(); - if ( elems == null ) { - return null; - } - else if ( elems.length == 1 ) { - return elems[0].getName(); - } - else { - throw new HttpClientException("Multiple Content-Type headers"); - } - } - return null; - } - private List

myHeaders; } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java b/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java index e9aab45..120a48b 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java @@ -12,9 +12,10 @@ import java.io.InputStream; import java.util.HashSet; import java.util.Set; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; + import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.message.BasicHeader; import org.expath.httpclient.ContentType; import org.expath.httpclient.HeaderSet; import org.expath.httpclient.HttpClientException; @@ -31,19 +32,17 @@ * * @author Florent Georges */ -public class BodyFactory -{ +public class BodyFactory { // TODO: Take new methods into account (XHTML, BASE64 and HEX). - public static HttpRequestBody makeRequestBody(Element elem, Sequence bodies, String ns) - throws HttpClientException - { + public static HttpRequestBody makeRequestBody(final Element elem, final Sequence bodies, final String ns) + throws HttpClientException { // method is got from @method if any... Type method = parseMethod(elem); // ...or from @media-type if no @method - if ( method == null ) { + if (method == null) { method = parseType(elem); } - switch ( method ) { + switch (method) { case MULTIPART: return new MultipartRequestBody(elem, bodies, ns); case SRC: @@ -61,36 +60,33 @@ public static HttpRequestBody makeRequestBody(Element elem, Sequence bodies, Str } } - public static HttpResponseBody makeResponseBody(Result result, ContentType type, HttpConnection conn) - throws HttpClientException - { - if ( type == null ) { + public static HttpResponseBody makeResponseBody(final Result result, final ContentType type, final HttpConnection conn) + throws HttpClientException { + if (type == null) { // it is legitimate to not have a body in a response; for instance // on a "304 Not Modified" return null; } String t = type.getType(); - if ( t == null ) { + if (t == null) { return null; } - InputStream in = conn.getResponseStream(); - if ( in == null ) { + final InputStream in = conn.getResponseStream(); + if (in == null) { return null; } - if ( t.startsWith("multipart/") ) { - return new MultipartResponseBody(result, in, type, conn); - } - else { + if (t.startsWith("multipart/")) { + return new MultipartResponseBody(result, in, type); + } else { return makeResponsePart(result, null, in, type); } } // package-level to be used within MultipartResponseBody ctor // TODO: Take new methods into account (XHTML, BASE64 and HEX). - static HttpResponseBody makeResponsePart(Result result, HeaderSet headers, InputStream in, ContentType ctype) - throws HttpClientException - { - switch ( parseType(ctype) ) { + static HttpResponseBody makeResponsePart(final Result result, final HeaderSet headers, final InputStream in, final ContentType ctype) + throws HttpClientException { + switch (parseType(ctype)) { case XML: // TODO: 'content_type' is the header Content-Type without any param // (i.e. "text/xml".) Should we keep this, or put the whole header @@ -107,8 +103,7 @@ static HttpResponseBody makeResponsePart(Result result, HeaderSet headers, Input } } - public static enum Type - { + public enum Type { XML, HTML, XHTML, @@ -120,20 +115,22 @@ public static enum Type SRC } - /** Media types that must be treated as text types (in addition to text/*). */ - private static Set TEXT_TYPES; + /** + * Media types that must be treated as text types (in addition to text/*). + */ + private static final Set TEXT_TYPES = new HashSet<>(); static { - TEXT_TYPES = new HashSet(); TEXT_TYPES.add("application/x-www-form-urlencoded"); TEXT_TYPES.add("application/xml-dtd"); } - /** Media types that must be treated as XML types (in addition to *+xml). */ - private static Set XML_TYPES; + /** + * Media types that must be treated as XML types (in addition to *+xml). + */ + private static final Set XML_TYPES = new HashSet<>(); static { // Doc: does not handle "application/xml-dtd" as XML // TODO: What about ".../xml-external-parsed-entity" ? - XML_TYPES = new HashSet(); XML_TYPES.add("text/xml"); XML_TYPES.add("application/xml"); XML_TYPES.add("text/xml-external-parsed-entity"); @@ -142,24 +139,19 @@ public static enum Type /** * Decode the content type from a MIME type string. - * + *

* TODO: Take new methods into account (XHTML, BASE64 and HEX). */ - private static Type parseType(String type) - { - if ( type.startsWith("multipart/") ) { + private static Type parseType(final String type) { + if (type.startsWith("multipart/")) { return Type.MULTIPART; - } - else if ( "text/html".equals(type) ) { + } else if ("text/html".equals(type)) { return Type.HTML; - } - else if ( type.endsWith("+xml") || XML_TYPES.contains(type) ) { + } else if (type.endsWith("+xml") || XML_TYPES.contains(type)) { return Type.XML; - } - else if ( type.startsWith("text/") || TEXT_TYPES.contains(type) ) { + } else if (type.startsWith("text/") || TEXT_TYPES.contains(type)) { return Type.TEXT; - } - else { + } else { return Type.BINARY; } } @@ -167,28 +159,24 @@ else if ( type.startsWith("text/") || TEXT_TYPES.contains(type) ) { /** * Look for the header COntent-Type in a header set and decode it. */ - public static Type parseType(HeaderSet headers) - throws HttpClientException - { - Header h = headers.getFirstHeader("Content-Type"); - if ( h == null ) { + public static Type parseType(final HeaderSet headers) throws HttpClientException { + final Header h = headers.getFirstHeader("Content-Type"); + if (h == null) { throw new HttpClientException("impossible to find the content type"); } - ContentType ct = new ContentType(h); + final ContentType ct = new ContentType(h); return parseType(ct); } /** * Decode the content type from a ContentType object. */ - public static Type parseType(ContentType type) - throws HttpClientException - { - if ( type == null ) { + public static Type parseType(final ContentType type) throws HttpClientException { + if (type == null) { throw new HttpClientException("impossible to find the content type"); } - String t = type.getType(); - if ( t == null ) { + final String t = type.getType(); + if (t == null) { throw new HttpClientException("impossible to find the content type"); } return parseType(t); @@ -197,74 +185,67 @@ public static Type parseType(ContentType type) /** * Parse the @media-type from a http:body or http:multipart element. */ - private static Type parseType(Element elem) - throws HttpClientException - { - String local = elem.getLocalName(); - if ( "multipart".equals(local) ) { + private static Type parseType(final Element elem) throws HttpClientException { + final String local = elem.getLocalName(); + if ("multipart".equals(local)) { return Type.MULTIPART; - } - else if ( ! "body".equals(local) ) { + } else if (!"body".equals(local)) { throw new HttpClientException("INTERNAL ERROR: cannot happen, checked before"); - } - else { - if ( elem.getAttribute("src") != null ) { + } else { + if (elem.getAttribute("src") != null) { return Type.SRC; } - String content_type = elem.getAttribute("media-type"); - if ( content_type == null ) { + final String mediaType = elem.getAttribute("media-type"); + if (mediaType == null) { throw new HttpClientException("@media-type is not set on http:body"); } - Type type = parseType(HeaderSet.getValueWithoutParam(content_type)); - if ( type == Type.MULTIPART ) { - String msg = "multipart type not allowed for http:body: " + content_type; - throw new HttpClientException(msg); + final Header mediaTypeHeader = new BasicHeader("Media-Type", mediaType); + final HeaderElement[] mediaTypeHeaderElems = mediaTypeHeader.getElements(); + if (mediaTypeHeaderElems == null || mediaTypeHeaderElems.length == 0) { + throw new HttpClientException("@media-type is not set on http:body"); + } else if (mediaTypeHeaderElems.length > 1) { + throw new HttpClientException("Multiple @media-type internet media types present"); + } else { + final Type type = parseType(mediaTypeHeaderElems[0].getName()); + if (type == Type.MULTIPART) { + final String msg = "multipart type not allowed for http:body: " + mediaType; + throw new HttpClientException(msg); + } + return type; } - return type; } } /** * Parse the @method from a http:body or http:multipart element. - * + *

* Return null if there is no @method. */ - private static Type parseMethod(Element elem) - throws HttpClientException - { - String m = elem.getAttribute("method"); - if ( m == null ) { + private static Type parseMethod(final Element elem) throws HttpClientException { + final String m = elem.getAttribute("method"); + if (m == null) { return null; - } - else if ( "xml".equals(m) ) { + } else if ("xml".equals(m)) { return Type.XML; - } - else if ( "html".equals(m) ) { + } else if ("html".equals(m)) { return Type.HTML; - } - else if ( "xhtml".equals(m) ) { + } else if ("xhtml".equals(m)) { return Type.XHTML; - } - else if ( "text".equals(m) ) { + } else if ("text".equals(m)) { return Type.TEXT; - } - else if ( "binary".equals(m) ) { + } else if ("binary".equals(m)) { return Type.BINARY; } // FIXME: The spec says "binary", but I think we need "base64" and "hex" // instead (or in addition, if "binary" is left implementation-defined). - else if ( "base64".equals(m) ) { + else if ("base64".equals(m)) { return Type.BASE64; - } - else if ( "hex".equals(m) ) { + } else if ("hex".equals(m)) { return Type.HEX; - } - else { + } else { throw new HttpClientException("Incorrect value for @method: " + m); } } - - private static Log LOG = LogFactory.getLog(BodyFactory.class); } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java index 3e8847d..d65acc9 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java @@ -87,7 +87,7 @@ private ContentType getContentType(HeaderSet headers) } } else { - return new ContentType(myOverrideType, null); + return new ContentType(myOverrideType, null, null); } } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java index f4a4984..383f362 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java @@ -15,6 +15,7 @@ import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.Header; @@ -25,7 +26,6 @@ import org.expath.httpclient.ContentType; import org.expath.httpclient.HeaderSet; import org.expath.httpclient.HttpClientException; -import org.expath.httpclient.HttpConnection; import org.expath.httpclient.HttpResponseBody; import org.expath.httpclient.model.Result; import org.expath.httpclient.model.TreeBuilder; @@ -36,80 +36,75 @@ * * @author Florent Georges */ -public class MultipartResponseBody - implements HttpResponseBody -{ - public MultipartResponseBody(Result result, InputStream in, ContentType type, HttpConnection conn) - throws HttpClientException - { - myContentType = type; - myParts = new ArrayList(); - Header h = conn.getResponseHeaders().getFirstHeader("Content-Type"); - if ( h == null ) { +public class MultipartResponseBody implements HttpResponseBody { + + public MultipartResponseBody(final Result result, final InputStream in, final ContentType type) + throws HttpClientException { + if (type == null || type.getType() == null) { throw new HttpClientException("No content type"); } + + myContentType = type; + myParts = new ArrayList<>(); + myBoundary = type.getBoundary(); - if ( myBoundary == null ) { + if (myBoundary == null) { throw new HttpClientException("No boundary"); } try { - analyzeParts(result, in, h.getValue()); - } - catch ( IOException ex ) { + analyzeParts(result, in); + } catch (IOException ex) { throw new HttpClientException("error reading the response stream", ex); } } @Override - public void outputBody(TreeBuilder b) - throws HttpClientException - { + public void outputBody(final TreeBuilder b) throws HttpClientException { try { b.startElem("multipart"); b.attribute("media-type", myContentType.getValue()); b.attribute("boundary", myBoundary); b.startContent(); - for ( HttpResponseBody part : myParts ) { + for (final HttpResponseBody part : myParts) { part.outputBody(b); } b.endElem(); - } - catch ( ToolsException ex ) { + } catch (final ToolsException ex) { throw new HttpClientException("Error building the body", ex); } } - private void analyzeParts(Result result, InputStream in, String type) - throws IOException - , HttpClientException - { - MimeTokenStream parser = new MimeTokenStream(); - parser.parseHeadless(in, type); + private void analyzeParts(final Result result, final InputStream in) throws IOException, HttpClientException { + final MimeTokenStream parser = new MimeTokenStream(); + + final String contentType; + if (myContentType.getCharset() != null) { + contentType = myContentType.getType() + "; charset=" + myContentType.getCharset(); + } else { + contentType = myContentType.getType(); + } + parser.parseHeadless(in, contentType); try { HeaderSet headers = null; - for ( EntityState state = parser.getState(); - state != EntityState.T_END_OF_STREAM; - state = parser.next() ) - { - if ( state == EntityState.T_START_HEADER ) { + for (EntityState state = parser.getState(); + state != EntityState.T_END_OF_STREAM; + state = parser.next()) { + if (state == EntityState.T_START_HEADER) { headers = new HeaderSet(); } handleParserState(result, parser, headers); } - } - catch ( MimeException ex ) { + } catch (final MimeException ex) { throw new HttpClientException("The response content is ill-formed.", ex); } } - private void handleParserState(Result result, MimeTokenStream parser, HeaderSet headers) - throws HttpClientException - { - EntityState state = parser.getState(); - if ( LOG.isDebugEnabled() ) { + private void handleParserState(final Result result, final MimeTokenStream parser, final HeaderSet headers) throws HttpClientException { + final EntityState state = parser.getState(); + if (LOG.isDebugEnabled()) { LOG.debug(MimeTokenStream.stateToString(state)); } - switch ( state ) { + switch (state) { // It seems that in a headless parsing, END_HEADER appears // right after START_MESSAGE (without the corresponding // START_HEADER). So if headers == null, we can just ignore @@ -118,17 +113,17 @@ private void handleParserState(Result result, MimeTokenStream parser, HeaderSet // TODO: Just ignore anyway...? break; case T_FIELD: - Field f = parser.getField(); - if ( LOG.isDebugEnabled() ) { + final Field f = parser.getField(); + if (LOG.isDebugEnabled()) { LOG.debug(" field: " + f); } headers.add(f.getName(), parseFieldBody(f)); break; case T_BODY: - if ( LOG.isDebugEnabled() ) { + if (LOG.isDebugEnabled()) { LOG.debug(" body desc: " + parser.getBodyDescriptor()); } - HttpResponseBody b = makeResponsePart(result, headers, parser); + final HttpResponseBody b = makeResponsePart(result, headers, parser); myParts.add(b); break; // START_HEADER is handled in the calling analyzeParts() @@ -148,21 +143,19 @@ private void handleParserState(Result result, MimeTokenStream parser, HeaderSet // should discover slowly that we should probably just // ignore some of them. default: - String s = MimeTokenStream.stateToString(state); + final String s = MimeTokenStream.stateToString(state); throw new HttpClientException("Unknown parsing state: " + s); } } - private String parseFieldBody(Field f) - throws HttpClientException - { + private String parseFieldBody(final Field f) { // try { - // WHy did I use AbstractField in the first place? - String b = f.getBody() /* AbstractField.parse(f.getRaw()).getBody() */; - if ( LOG.isDebugEnabled() ) { - LOG.debug("Field: " + f.getName() + ": [" + b + "]"); - } - return b; + // WHy did I use AbstractField in the first place? + final String b = f.getBody() /* AbstractField.parse(f.getRaw()).getBody() */; + if (LOG.isDebugEnabled()) { + LOG.debug("Field: " + f.getName() + ": [" + b + "]"); + } + return b; // } // catch ( MimeException ex ) { // LOG.error("Field value parsing error (" + f + ")", ex); @@ -170,40 +163,35 @@ private String parseFieldBody(Field f) // } } - private HttpResponseBody makeResponsePart(Result result, HeaderSet headers, MimeTokenStream parser) - throws HttpClientException - { - Header h = headers.getFirstHeader("Content-Type"); - if ( h == null ) { + private HttpResponseBody makeResponsePart(final Result result, final HeaderSet headers, final MimeTokenStream parser) + throws HttpClientException { + final Header h = headers.getFirstHeader("Content-Type"); + if (h == null) { throw new HttpClientException("impossible to find the content type"); } - ContentType type = new ContentType(h); + final ContentType type = new ContentType(h); try { switch (BodyFactory.parseType(type)) { case XML: { - // TODO: 'content_type' is the header Content-Type without any - // param (i.e. "text/xml".) Should we keep this, or put the - // whole header (i.e. "text/xml; charset=utf-8")? (and for - // other types as well...) - Reader in = parser.getReader(); + final Reader in = parser.getReader(); return new XmlResponseBody(result, in, type, headers, false); } case HTML: { - Reader in = parser.getReader(); + final Reader in = parser.getReader(); return new XmlResponseBody(result, in, type, headers, true); } case TEXT: { - Reader in = parser.getReader(); + final Reader in = parser.getReader(); return new TextResponseBody(result, in, type, headers); } case BINARY: { - InputStream in = parser.getInputStream(); + final InputStream in = parser.getInputStream(); return new BinaryResponseBody(result, in, type, headers); } default: throw new HttpClientException("INTERNAL ERROR: cannot happen"); } - } catch (UnsupportedEncodingException ex) { + } catch (final UnsupportedEncodingException ex) { throw new HttpClientException("Unable to parse response part", ex); } } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java index b36c499..52b057f 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java @@ -11,7 +11,12 @@ import java.io.*; import java.nio.charset.Charset; +import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.StandardCharsets; +import java.nio.charset.UnsupportedCharsetException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; import org.expath.httpclient.ContentType; import org.expath.httpclient.HeaderSet; @@ -21,38 +26,49 @@ import org.expath.httpclient.model.TreeBuilder; import org.expath.tools.ToolsException; +import static org.expath.httpclient.ContentType.DEFAULT_HTTP_CHARSET; + /** * A text body in the response. * * @author Florent Georges */ -public class TextResponseBody - implements HttpResponseBody -{ - public TextResponseBody(Result result, InputStream in, ContentType type, HeaderSet headers) - throws HttpClientException - { +public class TextResponseBody implements HttpResponseBody { + + public TextResponseBody(final Result result, final InputStream in, final ContentType type, final HeaderSet headers) + throws HttpClientException { myContentType = type; myHeaders = headers; - // TODO: ... - final Charset charset = StandardCharsets.UTF_8; - final Reader reader = new InputStreamReader(in, charset); - result.add(reader); + + final Charset contentCharset; + if (type.getCharset() != null) { + contentCharset = Charset.forName(type.getCharset()); + } else { + contentCharset = DEFAULT_HTTP_CHARSET; + } + + final Reader reader = new InputStreamReader(in, contentCharset); + result.add(reader, contentCharset); } - public TextResponseBody(Result result, Reader in, ContentType type, HeaderSet headers) - throws HttpClientException - { + public TextResponseBody(final Result result, final Reader in, final ContentType type, final HeaderSet headers) + throws HttpClientException { myContentType = type; myHeaders = headers; - result.add(in); + + final Charset contentCharset; + if (type.getCharset() != null) { + contentCharset = Charset.forName(type.getCharset()); + } else { + contentCharset = DEFAULT_HTTP_CHARSET; + } + + result.add(in, contentCharset); } @Override - public void outputBody(TreeBuilder b) - throws HttpClientException - { - if ( myHeaders != null ) { + public void outputBody(final TreeBuilder b) throws HttpClientException { + if (myHeaders != null) { b.outputHeaders(myHeaders); } try { @@ -61,14 +77,13 @@ public void outputBody(TreeBuilder b) // TODO: Support other attributes as well? b.startContent(); b.endElem(); - } - catch ( ToolsException ex ) { + } catch (ToolsException ex) { throw new HttpClientException("Error building the body", ex); } } - private ContentType myContentType; - private HeaderSet myHeaders; + private final ContentType myContentType; + private final HeaderSet myHeaders; } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java index 4a65e16..8bfc522 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java @@ -18,6 +18,7 @@ import javax.xml.transform.Source; import javax.xml.transform.sax.SAXSource; import javax.xml.transform.stream.StreamSource; + import org.ccil.cowan.tagsoup.Parser; import org.expath.httpclient.ContentType; import org.expath.httpclient.HeaderSet; @@ -29,59 +30,60 @@ import org.xml.sax.InputSource; import org.xml.sax.SAXException; +import static org.expath.httpclient.ContentType.DEFAULT_HTTP_CHARSET; + /** * An XML body in the response. * * @author Florent Georges */ -public class XmlResponseBody - implements HttpResponseBody -{ - public XmlResponseBody(Result result, InputStream in, ContentType type, HeaderSet headers, boolean html) - throws HttpClientException - { - // TODO: ... - final Charset charset = StandardCharsets.UTF_8; - final Reader reader = new InputStreamReader(in, charset); +public class XmlResponseBody implements HttpResponseBody { + public XmlResponseBody(final Result result, final InputStream in, final ContentType type, final HeaderSet headers, final boolean html) + throws HttpClientException { + + final Charset contentCharset; + if (type.getCharset() != null) { + contentCharset = Charset.forName(type.getCharset()); + } else { + contentCharset = DEFAULT_HTTP_CHARSET; + } + + final Reader reader = new InputStreamReader(in, contentCharset); init(result, reader, type, headers, html); } - public XmlResponseBody(Result result, Reader in, ContentType type, HeaderSet headers, boolean html) - throws HttpClientException - { + public XmlResponseBody(final Result result, final Reader in, final ContentType type, final HeaderSet headers, final boolean html) + throws HttpClientException { init(result, in, type, headers, html); } private void init(final Result result, final Reader in, final ContentType type, final HeaderSet headers, final boolean html) - throws HttpClientException - { + throws HttpClientException { myContentType = type; myHeaders = headers; String sys_id = "TODO-find-a-useful-systemId"; try { Source src; - if ( html ) { - Parser parser = new Parser(); + if (html) { + final Parser parser = new Parser(); parser.setFeature(Parser.namespacesFeature, true); parser.setFeature(Parser.namespacePrefixesFeature, true); - InputSource input = new InputSource(in); + final InputSource input = new InputSource(in); src = new SAXSource(parser, input); src.setSystemId(sys_id); } else { src = new StreamSource(in, sys_id); } result.add(src); - } - catch (SAXException ex) { + } catch (SAXException ex) { throw new HttpClientException("error parsing result HTML", ex); } } @Override - public void outputBody(TreeBuilder b) - throws HttpClientException - { - if ( myHeaders != null ) { + public void outputBody(final TreeBuilder b) + throws HttpClientException { + if (myHeaders != null) { b.outputHeaders(myHeaders); } try { @@ -90,8 +92,7 @@ public void outputBody(TreeBuilder b) // TODO: Support other attributes as well? b.startContent(); b.endElem(); - } - catch ( ToolsException ex ) { + } catch (ToolsException ex) { throw new HttpClientException("Error building the body", ex); } } diff --git a/http-client-java/src/main/java/org/expath/httpclient/model/Result.java b/http-client-java/src/main/java/org/expath/httpclient/model/Result.java index cd1c135..642bddb 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/model/Result.java +++ b/http-client-java/src/main/java/org/expath/httpclient/model/Result.java @@ -11,7 +11,9 @@ import java.io.Reader; import java.io.InputStream; +import java.nio.charset.Charset; import javax.xml.transform.Source; + import org.expath.httpclient.HttpClientException; import org.expath.httpclient.HttpResponse; @@ -70,7 +72,7 @@ public Result makeNewResult() * @param string The string to add to the result sequence. * @throws HttpClientException If any error occurs. */ - public void add(Reader string) + public void add(Reader string, Charset encoding) throws HttpClientException; /** diff --git a/http-client-saxon/src/main/java/org/expath/httpclient/saxon/SaxonResult.java b/http-client-saxon/src/main/java/org/expath/httpclient/saxon/SaxonResult.java index 34eeb28..9a5e676 100644 --- a/http-client-saxon/src/main/java/org/expath/httpclient/saxon/SaxonResult.java +++ b/http-client-saxon/src/main/java/org/expath/httpclient/saxon/SaxonResult.java @@ -10,6 +10,7 @@ package org.expath.httpclient.saxon; import java.io.*; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import javax.xml.transform.Source; @@ -51,7 +52,7 @@ public Result makeNewResult() } @Override - public void add(Reader reader) + public void add(Reader reader, Charset charset) throws HttpClientException { try(final BufferedReader buf_in = new BufferedReader(reader)) { From 225fa86af9b4ca85fbd9d765d2b62ef66b256ea1 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 11 Oct 2018 14:57:46 +0500 Subject: [PATCH 22/27] [bugfix] Preemptive Authentication was previously not working --- .../org/expath/httpclient/HttpClient.java | 17 ++--- .../org/expath/httpclient/HttpConnection.java | 1 + .../org/expath/httpclient/HttpRequest.java | 2 + .../httpclient/impl/ApacheHttpConnection.java | 70 ++++++++++++++----- .../httpclient/impl/HttpRequestImpl.java | 12 ++++ .../expath/httpclient/impl/RequestParser.java | 9 +-- 6 files changed, 74 insertions(+), 37 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/HttpClient.java b/http-client-java/src/main/java/org/expath/httpclient/HttpClient.java index 4de2782..7ee5233 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HttpClient.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HttpClient.java @@ -118,19 +118,14 @@ private Result sendOnce(Result result, URI uri, HttpRequest request, RequestPars { HttpConnection conn = new ApacheHttpConnection(uri); try { - if ( parser.getSendAuth() ) { + HttpResponse response = request.send(result, conn, parser.getCredentials()); + if ( response.getStatus() == 401 ) { + conn.disconnect(); + conn = new ApacheHttpConnection(uri); + // create a new result, and throw the old one away + result = result.makeNewResult(); request.send(result, conn, parser.getCredentials()); } - else { - HttpResponse response = request.send(result, conn, null); - if ( response.getStatus() == 401 ) { - conn.disconnect(); - conn = new ApacheHttpConnection(uri); - // create a new result, and throw the old one away - result = result.makeNewResult(); - request.send(result, conn, parser.getCredentials()); - } - } } finally { conn.disconnect(); diff --git a/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java index 7ce5cea..7ea9632 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HttpConnection.java @@ -36,6 +36,7 @@ public void setRequestMethod(String method, boolean with_content) public void setTimeout(int seconds); public void setGzip(boolean gzip); public void setChunked(boolean chunked); + public void setPreemptiveAuthentication(boolean preemptiveAuthentication); // responses... public int getResponseStatus() throws HttpClientException; diff --git a/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java index c5da7ce..23b1a74 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java @@ -37,6 +37,8 @@ public void setBody(HttpRequestBody body) public void setGzip(boolean gzip); public boolean isChunked(); public void setChunked(boolean chunked); + boolean isPreemptiveAuthentication(); + void setPreemptiveAuthentication(final boolean preemptiveAuthentication); } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java index 04440b9..139173f 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/ApacheHttpConnection.java @@ -25,9 +25,11 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpVersion; +import org.apache.http.auth.AuthScheme; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; import org.apache.http.client.CookieStore; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.GzipCompressingEntity; @@ -41,6 +43,8 @@ import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentProducer; import org.apache.http.entity.EntityTemplate; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.auth.DigestScheme; import org.apache.http.impl.client.*; import org.apache.http.impl.conn.*; import org.apache.http.protocol.HTTP; @@ -254,6 +258,11 @@ public void setChunked(final boolean chunked) { myChunked = chunked; } + @Override + public void setPreemptiveAuthentication(final boolean preemptiveAuthentication) { + myPreemptiveAuthentication = preemptiveAuthentication; + } + /** * Check the method name does match the HTTP/1.1 production rules. * @@ -379,46 +388,70 @@ private CloseableHttpClient makeClient() { * Set the credentials on the client, based on the {@link HttpCredentials} object. */ private HttpClientContext setCredentials(HttpCredentials cred) - throws HttpClientException - { + throws HttpClientException { final HttpClientContext clientContext = HttpClientContext.create(); - if ( cred == null ) { + if (cred == null) { return clientContext; } - URI uri = myRequest.getURI(); + final URI uri = myRequest.getURI(); + final String scheme = uri.getScheme(); int port = uri.getPort(); - if ( port == -1 ) { - String scheme = uri.getScheme(); - if ( "http".equals(scheme) ) { + if (port == -1) { + if ("http".equals(scheme)) { port = 80; - } - else if ( "https".equals(scheme) ) { + } else if ("https".equals(scheme)) { port = 443; - } - else { + } else { throw new HttpClientException("Unknown scheme: " + uri); } } - String host = uri.getHost(); - String user = cred.getUser(); - String pwd = cred.getPwd(); - if ( LOG.isDebugEnabled() ) { - LOG.debug("Set credentials for " + host + ":" + port + final String host = uri.getHost(); + + final HttpHost targetHost = new HttpHost(host, port, scheme); + + final String user = cred.getUser(); + final String pwd = cred.getPwd(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Set credentials for " + targetHost.getHostName() + ":" + targetHost.getPort() + " - " + user + " - ***"); } final Credentials c = new UsernamePasswordCredentials(user, pwd); - final AuthScope scope = new AuthScope(host, port); + final AuthScope scope = new AuthScope(targetHost); - if(clientContext.getCredentialsProvider() == null) { + if (clientContext.getCredentialsProvider() == null) { clientContext.setCredentialsProvider(new BasicCredentialsProvider()); } else { clientContext.getCredentialsProvider().clear(); } clientContext.getCredentialsProvider().setCredentials(scope, c); + + // force preemptive authentication? + // see - https://hc.apache.org/httpcomponents-client-ga/tutorial/html/authentication.html#d5e717 + if (myPreemptiveAuthentication) { + + // is there already an auth cache? + if (clientContext.getAuthCache() == null) { + // no, so create one + final AuthCache authCache = new BasicAuthCache(); + clientContext.setAuthCache(authCache); + } + + // set the auth cache scheme + final AuthScheme authScheme; + if (cred.getMethod().equals("DIGEST")) { + authScheme = new DigestScheme(); + } else { + authScheme = new BasicScheme(); + } + + clientContext.getAuthCache().put(targetHost, authScheme); + } + return clientContext; } @@ -569,6 +602,7 @@ private enum State { /** whether we should use gzip transfer encoding */ private boolean myGzip = false; private boolean myChunked = true; + private boolean myPreemptiveAuthentication = false; /** * The shared cookie store. diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java index d65acc9..79057ee 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java @@ -51,6 +51,7 @@ public HttpResponse send(Result result, HttpConnection conn, HttpCredentials cre conn.setGzip(true); } conn.setChunked(isChunked()); + conn.setPreemptiveAuthentication(isPreemptiveAuthentication()); conn.setFollowRedirect(myFollowRedirect); conn.connect(myBody, cred); @@ -197,6 +198,16 @@ public void setChunked(boolean chunked) { this.myChunked = chunked; } + @Override + public boolean isPreemptiveAuthentication() { + return myPreemptiveAuthentication; + } + + @Override + public void setPreemptiveAuthentication(final boolean preemptiveAuthentication) { + this.myPreemptiveAuthentication = preemptiveAuthentication; + } + private String myMethod; private String myHref; private String myHttpVer; @@ -206,6 +217,7 @@ public void setChunked(boolean chunked) { private Integer myTimeout = null; private boolean myGzip = false; private Boolean myChunked = null; + private boolean myPreemptiveAuthentication = false; private HeaderSet myHeaders; private HttpRequestBody myBody; private static final Log LOG = LogFactory.getLog(HttpRequestImpl.class); diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java index c6117aa..1b91572 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java @@ -62,11 +62,6 @@ public HttpCredentials getCredentials() return myCredentials; } - public boolean getSendAuth() - { - return mySendAuth; - } - public HttpRequest parse(Sequence bodies, String href) throws HttpClientException { @@ -118,7 +113,7 @@ else if ( "auth-method".equals(local) ) { auth_method = a.getValue(); } else if ( "send-authorization".equals(local) ) { - mySendAuth = toBoolean(a); + req.setPreemptiveAuthentication(toBoolean(a)); } else if ( "override-media-type".equals(local) ) { req.setOverrideType(a.getValue()); @@ -285,8 +280,6 @@ private int toInteger(Attribute a) private String myOtherNs; /** User credentials in case of authentication (from @username, @password and @auth-method). */ private HttpCredentials myCredentials = null; - /** The value of @send-authorization. */ - private boolean mySendAuth = false; } From 6d5e8373b2e86aa9264dd545ba3569d10b3d1d70 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Tue, 23 Apr 2019 21:34:50 +0800 Subject: [PATCH 23/27] [bugfix] Previously when override-media-type was set and a charset was specified, the charset was ignored --- .../org/expath/httpclient/ContentType.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/ContentType.java b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java index 0c6d71c..d97f010 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/ContentType.java +++ b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java @@ -33,8 +33,22 @@ public class ContentType { public ContentType(final String type, final String charset, final String boundary) { myHeader = null; myType = type; - myCharset = charset; - myBoundary = boundary; + + if (charset != null) { + myCharset = charset; + } else if (type.contains("charset=")) { + myCharset = type.replaceFirst(".+charset=([^;\\s]+).*", "$1"); + } else { + myCharset = null; + } + + if (boundary != null) { + myBoundary = boundary; + } else if (type.contains("boundary=")) { + myBoundary = type.replaceFirst(".+boundary=([^;\\s]+).*", "$1"); + } else { + myBoundary = null; + } } public ContentType(final Header h) throws HttpClientException { From b9e5a3f5f3adf0f47fa74d3efe1baf6b32f90c63 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 24 Apr 2019 13:34:33 +0800 Subject: [PATCH 24/27] [feature] Introduce default-charset parameter for http:request --- .../org/expath/httpclient/ContentType.java | 165 +++++++++--------- .../org/expath/httpclient/HttpRequest.java | 1 + .../expath/httpclient/impl/BodyFactory.java | 4 +- .../httpclient/impl/HttpRequestImpl.java | 22 +-- .../impl/MultipartResponseBody.java | 2 +- .../expath/httpclient/impl/RequestParser.java | 3 + 6 files changed, 103 insertions(+), 94 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/ContentType.java b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java index d97f010..6869ee0 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/ContentType.java +++ b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java @@ -31,77 +31,93 @@ public class ContentType { public static final Charset DEFAULT_HTTP_CHARSET = StandardCharsets.ISO_8859_1; public ContentType(final String type, final String charset, final String boundary) { - myHeader = null; - myType = type; - - if (charset != null) { - myCharset = charset; - } else if (type.contains("charset=")) { - myCharset = type.replaceFirst(".+charset=([^;\\s]+).*", "$1"); - } else { - myCharset = null; - } - - if (boundary != null) { - myBoundary = boundary; - } else if (type.contains("boundary=")) { - myBoundary = type.replaceFirst(".+boundary=([^;\\s]+).*", "$1"); - } else { - myBoundary = null; - } + this.myType = type; + this.myCharset = charset; + this.myBoundary = boundary; } - public ContentType(final Header h) throws HttpClientException { - if (h == null) { - throw new HttpClientException("Header is null"); - } - if (!"Content-Type".equalsIgnoreCase(h.getName())) { - throw new HttpClientException("Header is not content type"); - } - - this.myHeader = h; + public static ContentType parse(@Nullable final Header header, @Nullable final String overrideType, @Nullable final String defaultCharset) throws HttpClientException { + final String type; + final String charset; + final String boundary; + + if (overrideType != null) { + // get the internet media type from the override + type = extractMediaTypeFromContentType(overrideType); + + // does the override contain a charset? + if (overrideType.indexOf("charset=") > -1) { + // get the charset from the override + charset = overrideType.replaceFirst(".+charset=([^;\\s]+).*", "$1"); + } else { + // get the charset from the header or the default + if (header == null || !"Content-Type".equalsIgnoreCase(header.getName())) { + throw new HttpClientException("Header is not content type"); + } + final HeaderElement[] headerElements = header.getElements(); + if (headerElements.length > 1) { + throw new HttpClientException("Multiple Content-Type headers"); + } - final HeaderElement[] elems = myHeader.getElements(); - if (elems == null || elems.length == 0) { - this.myType = null; - } else if (elems.length > 1) { - throw new HttpClientException("Multiple Content-Type headers"); - } else { - this.myType = elems[0].getName(); - } + final NameValuePair headerCharset = headerElements[0].getParameterByName("charset"); + if (headerCharset != null) { + charset = headerCharset.getValue(); + } else { + charset = defaultCharset; + } + } - String charset = null; - String boundary = null; - if (elems != null) { - for (final HeaderElement e : elems) { - final NameValuePair nvpCharset = e.getParameterByName("charset"); - if (nvpCharset != null) { - charset = nvpCharset.getValue(); + // does the override contain a boundary? + if (overrideType.indexOf("boundary=") > -1) { + boundary = overrideType.replaceFirst(".+boundary=([^;\\s]+).*", "$1"); + } else { + // get the boundary from the header or null + if (header == null || !"Content-Type".equalsIgnoreCase(header.getName())) { + throw new HttpClientException("Header is not content type"); } - final NameValuePair nvpBoundary = e.getParameterByName("boundary"); - if (nvpBoundary != null) { - boundary = nvpBoundary.getValue(); + final HeaderElement[] headerElements = header.getElements(); + if (headerElements.length > 1) { + throw new HttpClientException("Multiple Content-Type headers"); } + + final NameValuePair headerBoundary = headerElements[0].getParameterByName("boundary"); + boundary = headerBoundary == null ? null : headerBoundary.getValue(); } - } - this.myCharset = charset; - this.myBoundary = boundary; - } - @Override - public String toString() { - if (myHeader == null) { - final StringBuilder builder = new StringBuilder("Content-Type: ").append(getValue()); - if (myCharset != null) { - builder.append("; charset=").append(myCharset); + } else { + // get the internet media type from the header + if (header == null || !"Content-Type".equalsIgnoreCase(header.getName())) { + throw new HttpClientException("Header is not content type"); } - if (myBoundary != null) { - builder.append("; boundary=").append(myBoundary); + final HeaderElement[] headerElements = header.getElements(); + if (headerElements.length > 1) { + throw new HttpClientException("Multiple Content-Type headers"); } - return builder.toString(); + type = extractMediaTypeFromContentType(header.getValue()); + + // get the charset from the header or the default + final NameValuePair headerCharset = headerElements[0].getParameterByName("charset"); + if (headerCharset != null) { + charset = headerCharset.getValue(); + } else { + charset = defaultCharset; + } + + // get the boundary from the header + final NameValuePair headerBoundary = headerElements[0].getParameterByName("boundary"); + boundary = headerBoundary == null ? null : headerBoundary.getValue(); + } + + return new ContentType(type, charset, boundary); + } + + private static String extractMediaTypeFromContentType(final String contentType) { + final int idxParamSeparator = contentType.indexOf(';'); + if (idxParamSeparator > -1) { + return contentType.substring(0, idxParamSeparator); } else { - return myHeader.toString(); + return contentType; } } @@ -122,29 +138,22 @@ public String getBoundary() { @Nullable public String getValue() { - // TODO: Why did I add the boundary before...? -// if ( myHeader == null ) { -// StringBuilder b = new StringBuilder(); -// b.append(myType); -// if ( myBoundary != null ) { -// b.append("; boundary=\""); -// // TODO: Is that correct escaping sequence? -// b.append(myBoundary.replace("\"", "\\\"")); -// b.append("\""); -// } -// return b.toString(); -// } - if (myType != null) { - return myType; + final StringBuilder builder = new StringBuilder(myType); + if (myCharset != null) { + builder.append("; charset=").append(myCharset); } - if (myHeader != null) { - return myHeader.getValue(); - } else { - return null; + if (myBoundary != null) { + builder.append("; boundary=").append(myCharset); } + + return builder.toString(); + } + + @Override + public String toString() { + return getValue(); } - private final Header myHeader; private final String myType; private final String myCharset; private final String myBoundary; diff --git a/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java index 23b1a74..46a8030 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java +++ b/http-client-java/src/main/java/org/expath/httpclient/HttpRequest.java @@ -27,6 +27,7 @@ public HttpResponse send(Result result, HttpConnection conn, HttpCredentials cre public String getHttpVersion(); public void setHttpVersion(String ver) throws HttpClientException; + public void setDefaultCharset(String charset); public void setOverrideType(String type); public void setHeaders(HeaderSet headers); public void setBody(HttpRequestBody body) diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java b/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java index 120a48b..649d5bd 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/BodyFactory.java @@ -157,14 +157,14 @@ private static Type parseType(final String type) { } /** - * Look for the header COntent-Type in a header set and decode it. + * Look for the header Content-Type in a header set and decode it. */ public static Type parseType(final HeaderSet headers) throws HttpClientException { final Header h = headers.getFirstHeader("Content-Type"); if (h == null) { throw new HttpClientException("impossible to find the content type"); } - final ContentType ct = new ContentType(h); + final ContentType ct = ContentType.parse(h, null, null); return parseType(ct); } diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java index 79057ee..23ff4b5 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/HttpRequestImpl.java @@ -75,21 +75,11 @@ public HttpResponse send(Result result, HttpConnection conn, HttpCredentials cre return resp; } - private ContentType getContentType(HeaderSet headers) + private ContentType getContentType(final HeaderSet headers) throws HttpClientException { - if ( myOverrideType == null ) { - Header h = headers.getFirstHeader("Content-Type"); - if ( h == null ) { - return null; - } - else { - return new ContentType(h); - } - } - else { - return new ContentType(myOverrideType, null, null); - } + final Header header = headers.getFirstHeader("Content-Type"); + return ContentType.parse(header, myOverrideType, myDefaultCharset); } @Override @@ -137,6 +127,11 @@ else if ( HttpConstants.HTTP_1_1.equals(ver) ) { } } + @Override + public void setDefaultCharset(final String charset) { + myDefaultCharset = charset; + } + @Override public void setOverrideType(String type) { @@ -211,6 +206,7 @@ public void setPreemptiveAuthentication(final boolean preemptiveAuthentication) private String myMethod; private String myHref; private String myHttpVer; + private String myDefaultCharset; private String myOverrideType; private boolean myStatusOnly; private boolean myFollowRedirect = true; diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java index 383f362..61f11ba 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/MultipartResponseBody.java @@ -169,7 +169,7 @@ private HttpResponseBody makeResponsePart(final Result result, final HeaderSet h if (h == null) { throw new HttpClientException("impossible to find the content type"); } - final ContentType type = new ContentType(h); + final ContentType type = ContentType.parse(h, null, null); try { switch (BodyFactory.parseType(type)) { case XML: { diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java index 1b91572..6423baa 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/RequestParser.java @@ -115,6 +115,9 @@ else if ( "auth-method".equals(local) ) { else if ( "send-authorization".equals(local) ) { req.setPreemptiveAuthentication(toBoolean(a)); } + else if ( "default-charset".equals(local) ) { + req.setDefaultCharset(a.getValue()); + } else if ( "override-media-type".equals(local) ) { req.setOverrideType(a.getValue()); } From b369069c06695560a2d5077493b2cf75e14ef14c Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Wed, 1 May 2019 11:04:50 +0400 Subject: [PATCH 25/27] [bugfix] Default charset encoding for application/xml should be UTF-8 not ISO-8859-1 See https://tools.ietf.org/html/rfc7231#appendix-B See https://tools.ietf.org/html/rfc7303#section-2.2 See https://www.w3.org/TR/xml/#charencoding --- .../java/org/expath/httpclient/ContentType.java | 4 ---- .../expath/httpclient/impl/TextResponseBody.java | 13 ++++--------- .../org/expath/httpclient/impl/XmlResponseBody.java | 8 ++++---- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/http-client-java/src/main/java/org/expath/httpclient/ContentType.java b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java index 6869ee0..94af7b9 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/ContentType.java +++ b/http-client-java/src/main/java/org/expath/httpclient/ContentType.java @@ -14,8 +14,6 @@ import org.apache.http.NameValuePair; import javax.annotation.Nullable; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; /** * Represent a Content-Type header. @@ -28,8 +26,6 @@ */ public class ContentType { - public static final Charset DEFAULT_HTTP_CHARSET = StandardCharsets.ISO_8859_1; - public ContentType(final String type, final String charset, final String boundary) { this.myType = type; this.myCharset = charset; diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java index 52b057f..cc5c418 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/TextResponseBody.java @@ -11,12 +11,7 @@ import java.io.*; import java.nio.charset.Charset; -import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.StandardCharsets; -import java.nio.charset.UnsupportedCharsetException; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.annotation.Nullable; import org.expath.httpclient.ContentType; import org.expath.httpclient.HeaderSet; @@ -26,8 +21,6 @@ import org.expath.httpclient.model.TreeBuilder; import org.expath.tools.ToolsException; -import static org.expath.httpclient.ContentType.DEFAULT_HTTP_CHARSET; - /** * A text body in the response. * @@ -35,6 +28,8 @@ */ public class TextResponseBody implements HttpResponseBody { + public static final Charset DEFAULT_HTTP_TEXT_CHARSET = StandardCharsets.ISO_8859_1; + public TextResponseBody(final Result result, final InputStream in, final ContentType type, final HeaderSet headers) throws HttpClientException { myContentType = type; @@ -44,7 +39,7 @@ public TextResponseBody(final Result result, final InputStream in, final Content if (type.getCharset() != null) { contentCharset = Charset.forName(type.getCharset()); } else { - contentCharset = DEFAULT_HTTP_CHARSET; + contentCharset = DEFAULT_HTTP_TEXT_CHARSET; } final Reader reader = new InputStreamReader(in, contentCharset); @@ -60,7 +55,7 @@ public TextResponseBody(final Result result, final Reader in, final ContentType if (type.getCharset() != null) { contentCharset = Charset.forName(type.getCharset()); } else { - contentCharset = DEFAULT_HTTP_CHARSET; + contentCharset = DEFAULT_HTTP_TEXT_CHARSET; } result.add(in, contentCharset); diff --git a/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java b/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java index 8bfc522..400fe29 100644 --- a/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java +++ b/http-client-java/src/main/java/org/expath/httpclient/impl/XmlResponseBody.java @@ -12,7 +12,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import javax.xml.transform.Source; @@ -30,14 +29,15 @@ import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import static org.expath.httpclient.ContentType.DEFAULT_HTTP_CHARSET; - /** * An XML body in the response. * * @author Florent Georges */ public class XmlResponseBody implements HttpResponseBody { + + public static final Charset DEFAULT_HTTP_APPLICATION_XML_CHARSET = StandardCharsets.UTF_8; + public XmlResponseBody(final Result result, final InputStream in, final ContentType type, final HeaderSet headers, final boolean html) throws HttpClientException { @@ -45,7 +45,7 @@ public XmlResponseBody(final Result result, final InputStream in, final ContentT if (type.getCharset() != null) { contentCharset = Charset.forName(type.getCharset()); } else { - contentCharset = DEFAULT_HTTP_CHARSET; + contentCharset = DEFAULT_HTTP_APPLICATION_XML_CHARSET; } final Reader reader = new InputStreamReader(in, contentCharset); From 45b39a071d133306d073f2f4fd0d5e1341b8e4d8 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 2 May 2019 20:34:51 +0400 Subject: [PATCH 26/27] Should not have a relative path to expath-parent --- http-client-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-client-parent/pom.xml b/http-client-parent/pom.xml index 61f7ef7..b17f55c 100644 --- a/http-client-parent/pom.xml +++ b/http-client-parent/pom.xml @@ -7,7 +7,7 @@ org.expath expath-parent 1.0-SNAPSHOT - ../../expath-parent/pom.xml + org.expath.http.client From 3931545ab33b93d2e245df5bf1cec12835e96de3 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Thu, 2 May 2019 20:46:14 +0400 Subject: [PATCH 27/27] Update dependencies --- http-client-java/pom.xml | 7 +------ http-client-parent/pom.xml | 8 ++++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/http-client-java/pom.xml b/http-client-java/pom.xml index 4b6e971..33ac73b 100644 --- a/http-client-java/pom.xml +++ b/http-client-java/pom.xml @@ -64,12 +64,7 @@ org.apache.james apache-mime4j-core - 0.8.1 - - - org.apache.james - apache-mime4j-dom - 0.8.1 + 0.8.3 commons-logging diff --git a/http-client-parent/pom.xml b/http-client-parent/pom.xml index b17f55c..45a33a4 100644 --- a/http-client-parent/pom.xml +++ b/http-client-parent/pom.xml @@ -32,7 +32,7 @@ maven-assembly-plugin - 2.6 + 3.1.1 @@ -43,17 +43,17 @@ org.expath.tools tools-java - 1.0-SNAPSHOT + ${expath.tools.version} org.apache.httpcomponents httpcore - 4.4.9 + 4.4.11 org.apache.httpcomponents httpclient - 4.5.4 + 4.5.8