View Javadoc
1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  
28  package org.apache.hc.client5.testing.sync;
29  
30  import java.io.ByteArrayInputStream;
31  import java.io.ByteArrayOutputStream;
32  import java.io.OutputStream;
33  import java.nio.charset.StandardCharsets;
34  import java.util.ArrayList;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.concurrent.CountDownLatch;
38  import java.util.concurrent.ExecutorService;
39  import java.util.concurrent.Executors;
40  import java.util.zip.Deflater;
41  import java.util.zip.GZIPOutputStream;
42  
43  import org.apache.hc.client5.http.classic.methods.HttpGet;
44  import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler;
45  import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
46  import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
47  import org.apache.hc.client5.testing.extension.sync.ClientProtocolLevel;
48  import org.apache.hc.client5.testing.extension.sync.TestClient;
49  import org.apache.hc.core5.http.HeaderElement;
50  import org.apache.hc.core5.http.HttpHost;
51  import org.apache.hc.core5.http.HttpStatus;
52  import org.apache.hc.core5.http.URIScheme;
53  import org.apache.hc.core5.http.io.HttpRequestHandler;
54  import org.apache.hc.core5.http.io.entity.EntityUtils;
55  import org.apache.hc.core5.http.io.entity.InputStreamEntity;
56  import org.apache.hc.core5.http.io.entity.StringEntity;
57  import org.apache.hc.core5.http.message.MessageSupport;
58  import org.junit.jupiter.api.Assertions;
59  import org.junit.jupiter.api.Test;
60  
61  /**
62   * Test case for how Content Codings are processed. By default, we want to do the right thing and
63   * require no intervention from the user of HttpClient, but we still want to let clients do their
64   * own thing if they so wish.
65   */
66  abstract  class TestContentCodings extends AbstractIntegrationTestBase {
67  
68      protected TestContentCodings(final URIScheme scheme) {
69          super(scheme, ClientProtocolLevel.STANDARD);
70      }
71  
72      /**
73       * Test for when we don't get an entity back; e.g. for a 204 or 304 response; nothing blows
74       * up with the new behaviour.
75       *
76       * @throws Exception
77       *             if there was a problem
78       */
79      @Test
80      void testResponseWithNoContent() throws Exception {
81          configureServer(bootstrap -> bootstrap
82                  .register("*", (request, response, context) -> response.setCode(HttpStatus.SC_NO_CONTENT)));
83  
84          final HttpHost target = startServer();
85  
86          final TestClient client = client();
87  
88          final HttpGet request = new HttpGet("/some-resource");
89          client.execute(target, request, response -> {
90              Assertions.assertEquals(HttpStatus.SC_NO_CONTENT, response.getCode());
91              Assertions.assertNull(response.getEntity());
92              return null;
93          });
94      }
95  
96      /**
97       * Test for when we are handling content from a server that has correctly interpreted RFC2616
98       * to return RFC1950 streams for {@code deflate} content coding.
99       *
100      * @throws Exception
101      */
102     @Test
103     void testDeflateSupportForServerReturningRfc1950Stream() throws Exception {
104         final String entityText = "Hello, this is some plain text coming back.";
105 
106         configureServer(bootstrap -> bootstrap
107                 .register("*", createDeflateEncodingRequestHandler(entityText, false)));
108 
109         final HttpHost target = startServer();
110 
111         final TestClient client = client();
112 
113         final HttpGet request = new HttpGet("/some-resource");
114         client.execute(target, request, response -> {
115             Assertions.assertEquals(entityText, EntityUtils.toString(response.getEntity()),
116                     "The entity text is correctly transported");
117             return null;
118         });
119     }
120 
121     /**
122      * Test for when we are handling content from a server that has incorrectly interpreted RFC2616
123      * to return RFC1951 streams for {@code deflate} content coding.
124      *
125      * @throws Exception
126      */
127     @Test
128     void testDeflateSupportForServerReturningRfc1951Stream() throws Exception {
129         final String entityText = "Hello, this is some plain text coming back.";
130 
131         configureServer(bootstrap -> bootstrap
132                 .register("*", createDeflateEncodingRequestHandler(entityText, true)));
133 
134         final HttpHost target = startServer();
135 
136         final TestClient client = client();
137 
138         final HttpGet request = new HttpGet("/some-resource");
139         client.execute(target, request, response -> {
140             Assertions.assertEquals(entityText, EntityUtils.toString(response.getEntity()),
141                     "The entity text is correctly transported");
142             return null;
143         });
144     }
145 
146     /**
147      * Test for a server returning gzipped content.
148      *
149      * @throws Exception
150      */
151     @Test
152     void testGzipSupport() throws Exception {
153         final String entityText = "Hello, this is some plain text coming back.";
154 
155         configureServer(bootstrap -> bootstrap
156                 .register("*", createGzipEncodingRequestHandler(entityText)));
157 
158         final HttpHost target = startServer();
159 
160         final TestClient client = client();
161 
162         final HttpGet request = new HttpGet("/some-resource");
163         client.execute(target, request, response -> {
164             Assertions.assertEquals(entityText, EntityUtils.toString(response.getEntity()),
165                     "The entity text is correctly transported");
166             return null;
167         });
168     }
169 
170     /**
171      * Try with a bunch of client threads, to check that it's thread-safe.
172      *
173      * @throws Exception
174      *             if there was a problem
175      */
176     @Test
177     void testThreadSafetyOfContentCodings() throws Exception {
178         final String entityText = "Hello, this is some plain text coming back.";
179 
180         configureServer(bootstrap -> bootstrap
181                 .register("*", createGzipEncodingRequestHandler(entityText)));
182 
183         final HttpHost target = startServer();
184 
185         final TestClient client = client();
186         final PoolingHttpClientConnectionManager connManager = client.getConnectionManager();
187 
188         /*
189          * Create a load of workers which will access the resource. Half will use the default
190          * gzip behaviour; half will require identity entity.
191          */
192         final int clients = 10;
193 
194         connManager.setMaxTotal(clients);
195 
196         final ExecutorService executor = Executors.newFixedThreadPool(clients);
197 
198         final CountDownLatch startGate = new CountDownLatch(1);
199         final CountDownLatch endGate = new CountDownLatch(clients);
200 
201         final List<WorkerTask> workers = new ArrayList<>();
202 
203         for (int i = 0; i < clients; ++i) {
204             workers.add(new WorkerTask(client, target, i % 2 == 0, startGate, endGate));
205         }
206 
207         for (final WorkerTask workerTask : workers) {
208 
209             /* Set them all in motion, but they will block until we call startGate.countDown(). */
210             executor.execute(workerTask);
211         }
212 
213         startGate.countDown();
214 
215         /* Wait for the workers to complete. */
216         endGate.await();
217 
218         for (final WorkerTask workerTask : workers) {
219             if (workerTask.isFailed()) {
220                 Assertions.fail("A worker failed");
221             }
222             Assertions.assertEquals(entityText, workerTask.getText());
223         }
224     }
225 
226     @Test
227     void testHttpEntityWriteToForGzip() throws Exception {
228         final String entityText = "Hello, this is some plain text coming back.";
229 
230         configureServer(bootstrap -> bootstrap
231                 .register("*", createGzipEncodingRequestHandler(entityText)));
232 
233         final HttpHost target = startServer();
234 
235         final TestClient client = client();
236 
237         final HttpGet request = new HttpGet("/some-resource");
238         client.execute(target, request, response -> {
239             final ByteArrayOutputStream out = new ByteArrayOutputStream();
240             response.getEntity().writeTo(out);
241             Assertions.assertEquals(entityText, out.toString("utf-8"));
242             return null;
243         });
244 
245     }
246 
247     @Test
248     void testHttpEntityWriteToForDeflate() throws Exception {
249         final String entityText = "Hello, this is some plain text coming back.";
250 
251         configureServer(bootstrap -> bootstrap
252                 .register("*", createDeflateEncodingRequestHandler(entityText, true)));
253 
254         final HttpHost target = startServer();
255 
256         final TestClient client = client();
257 
258         final HttpGet request = new HttpGet("/some-resource");
259         client.execute(target, request, response -> {
260             final ByteArrayOutputStream out = new ByteArrayOutputStream();
261             response.getEntity().writeTo(out);
262             Assertions.assertEquals(entityText, out.toString("utf-8"));
263             return out;
264         });
265     }
266 
267     @Test
268     void gzipResponsesWorkWithBasicResponseHandler() throws Exception {
269         final String entityText = "Hello, this is some plain text coming back.";
270 
271         configureServer(bootstrap -> bootstrap
272                 .register("*", createGzipEncodingRequestHandler(entityText)));
273 
274         final HttpHost target = startServer();
275 
276         final TestClient client = client();
277 
278         final HttpGet request = new HttpGet("/some-resource");
279         final String response = client.execute(target, request, new BasicHttpClientResponseHandler());
280         Assertions.assertEquals(entityText, response, "The entity text is correctly transported");
281     }
282 
283     @Test
284     void deflateResponsesWorkWithBasicResponseHandler() throws Exception {
285         final String entityText = "Hello, this is some plain text coming back.";
286 
287         configureServer(bootstrap -> bootstrap
288                 .register("*", createDeflateEncodingRequestHandler(entityText, false)));
289 
290         final HttpHost target = startServer();
291 
292         final TestClient client = client();
293 
294         final HttpGet request = new HttpGet("/some-resource");
295         final String response = client.execute(target, request, new BasicHttpClientResponseHandler());
296         Assertions.assertEquals(entityText, response, "The entity text is correctly transported");
297     }
298 
299     /**
300      * Creates a new {@link HttpRequestHandler} that will attempt to provide a deflate stream
301      * Content-Coding.
302      *
303      * @param entityText
304      *            the non-null String entity text to be returned by the server
305      * @param rfc1951
306      *            if true, then the stream returned will be a raw RFC1951 deflate stream, which
307      *            some servers return as a result of misinterpreting the HTTP 1.1 RFC. If false,
308      *            then it will return an RFC2616 compliant deflate encoded zlib stream.
309      * @return a non-null {@link HttpRequestHandler}
310      */
311     private HttpRequestHandler createDeflateEncodingRequestHandler(
312             final String entityText, final boolean rfc1951) {
313         return (request, response, context) -> {
314             response.setEntity(new StringEntity(entityText));
315             response.addHeader("Content-Type", "text/plain");
316             final Iterator<HeaderElement> it = MessageSupport.iterate(request, "Accept-Encoding");
317             while (it.hasNext()) {
318         final HeaderElement element = it.next();
319         if ("deflate".equalsIgnoreCase(element.getName())) {
320             response.addHeader("Content-Encoding", "deflate");
321 
322                 /* Gack. DeflaterInputStream is Java 6. */
323             // response.setEntity(new InputStreamEntity(new DeflaterInputStream(new
324             // ByteArrayInputStream(
325             // entityText.getBytes("utf-8"))), -1));
326             final byte[] uncompressed = entityText.getBytes(StandardCharsets.UTF_8);
327             final Deflater compressor = new Deflater(Deflater.DEFAULT_COMPRESSION, rfc1951);
328             compressor.setInput(uncompressed);
329             compressor.finish();
330             final byte[] output = new byte[100];
331             final int compressedLength = compressor.deflate(output);
332             final byte[] compressed = new byte[compressedLength];
333             System.arraycopy(output, 0, compressed, 0, compressedLength);
334             response.setEntity(new InputStreamEntity(
335                     new ByteArrayInputStream(compressed), compressedLength, null));
336             return;
337         }
338             }
339          };
340     }
341 
342     /**
343      * Returns an {@link HttpRequestHandler} implementation that will attempt to provide a gzip
344      * Content-Encoding.
345      *
346      * @param entityText
347      *            the non-null String entity to be returned by the server
348      * @return a non-null {@link HttpRequestHandler}
349      */
350     private HttpRequestHandler createGzipEncodingRequestHandler(final String entityText) {
351         return (request, response, context) -> {
352             response.setEntity(new StringEntity(entityText));
353             response.addHeader("Content-Type", "text/plain");
354             response.addHeader("Content-Type", "text/plain");
355             final Iterator<HeaderElement> it = MessageSupport.iterate(request, "Accept-Encoding");
356             while (it.hasNext()) {
357         final HeaderElement element = it.next();
358         if ("gzip".equalsIgnoreCase(element.getName())) {
359             response.addHeader("Content-Encoding", "gzip");
360 
361             /*
362              * We have to do a bit more work with gzip versus deflate, since
363              * Gzip doesn't appear to have an equivalent to DeflaterInputStream in
364              * the JDK.
365              *
366              * UPDATE: DeflaterInputStream is Java 6 anyway, so we have to do a bit
367              * of work there too!
368              */
369             final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
370             final OutputStream out = new GZIPOutputStream(bytes);
371 
372             final ByteArrayInputStream uncompressed = new ByteArrayInputStream(
373                     entityText.getBytes(StandardCharsets.UTF_8));
374 
375             final byte[] buf = new byte[60];
376 
377             int n;
378             while ((n = uncompressed.read(buf)) != -1) {
379                 out.write(buf, 0, n);
380             }
381 
382             out.close();
383 
384             final byte[] arr = bytes.toByteArray();
385             response.setEntity(new InputStreamEntity(new ByteArrayInputStream(arr),
386                     arr.length, null));
387 
388             return;
389         }
390             }
391          };
392     }
393 
394     /**
395      * Sub-ordinate task passed off to a different thread to be executed.
396      *
397      * @author jabley
398      *
399      */
400     class WorkerTask implements Runnable {
401 
402         private final CloseableHttpClient client;
403         private final HttpHost target;
404         private final HttpGet request;
405         private final CountDownLatch startGate;
406         private final CountDownLatch endGate;
407 
408         private boolean failed;
409         private String text;
410 
411         WorkerTask(final CloseableHttpClient client, final HttpHost target, final boolean identity, final CountDownLatch startGate, final CountDownLatch endGate) {
412             this.client = client;
413             this.target = target;
414             this.request = new HttpGet("/some-resource");
415             if (identity) {
416                 request.addHeader("Accept-Encoding", "identity");
417             }
418             this.startGate = startGate;
419             this.endGate = endGate;
420         }
421 
422         /**
423          * Returns the text of the HTTP entity.
424          *
425          * @return a String - may be null.
426          */
427         public String getText() {
428             return this.text;
429         }
430 
431         /**
432          * {@inheritDoc}
433          */
434         @Override
435         public void run() {
436             try {
437                 startGate.await();
438                 try {
439                     text = client.execute(target, request, response ->
440                             EntityUtils.toString(response.getEntity()));
441                 } catch (final Exception e) {
442                     failed = true;
443                 } finally {
444                     endGate.countDown();
445                 }
446             } catch (final InterruptedException ignore) {
447             }
448         }
449 
450         /**
451          * Returns true if this task failed, otherwise false.
452          *
453          * @return a flag
454          */
455         boolean isFailed() {
456             return this.failed;
457         }
458     }
459 }