View Javadoc

1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.net.URI;
31  import java.util.Date;
32  import java.util.HashMap;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.concurrent.Future;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.http.Header;
41  import org.apache.http.HeaderElement;
42  import org.apache.http.HttpHost;
43  import org.apache.http.HttpMessage;
44  import org.apache.http.HttpRequest;
45  import org.apache.http.HttpResponse;
46  import org.apache.http.HttpStatus;
47  import org.apache.http.HttpVersion;
48  import org.apache.http.ProtocolException;
49  import org.apache.http.ProtocolVersion;
50  import org.apache.http.RequestLine;
51  import org.apache.http.annotation.ThreadSafe;
52  import org.apache.http.client.ClientProtocolException;
53  import org.apache.http.client.cache.CacheResponseStatus;
54  import org.apache.http.client.cache.HeaderConstants;
55  import org.apache.http.client.cache.HttpCacheContext;
56  import org.apache.http.client.cache.HttpCacheEntry;
57  import org.apache.http.client.cache.HttpCacheStorage;
58  import org.apache.http.client.cache.ResourceFactory;
59  import org.apache.http.client.methods.HttpRequestWrapper;
60  import org.apache.http.client.methods.HttpUriRequest;
61  import org.apache.http.client.protocol.HttpClientContext;
62  import org.apache.http.client.utils.DateUtils;
63  import org.apache.http.concurrent.BasicFuture;
64  import org.apache.http.concurrent.FutureCallback;
65  import org.apache.http.conn.routing.HttpRoute;
66  import org.apache.http.impl.nio.client.HttpAsyncClients;
67  import org.apache.http.message.BasicHttpResponse;
68  import org.apache.http.nio.client.HttpAsyncClient;
69  import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
70  import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
71  import org.apache.http.nio.reactor.IOReactorException;
72  import org.apache.http.protocol.HTTP;
73  import org.apache.http.protocol.HttpContext;
74  import org.apache.http.util.Args;
75  import org.apache.http.util.EntityUtils;
76  import org.apache.http.util.VersionInfo;
77  
78  @ThreadSafe // So long as the responseCache implementation is threadsafe
79  public class CachingHttpAsyncClient implements HttpAsyncClient {
80  
81      private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
82  
83      private final AtomicLong cacheHits = new AtomicLong();
84      private final AtomicLong cacheMisses = new AtomicLong();
85      private final AtomicLong cacheUpdates = new AtomicLong();
86  
87      private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
88  
89      private final HttpAsyncClient backend;
90      private final HttpCache responseCache;
91      private final CacheValidityPolicy validityPolicy;
92      private final ResponseCachingPolicy responseCachingPolicy;
93      private final CachedHttpResponseGenerator responseGenerator;
94      private final CacheableRequestPolicy cacheableRequestPolicy;
95      private final CachedResponseSuitabilityChecker suitabilityChecker;
96  
97      private final ConditionalRequestBuilder conditionalRequestBuilder;
98  
99      private final long maxObjectSizeBytes;
100     private final boolean sharedCache;
101 
102     private final ResponseProtocolCompliance responseCompliance;
103     private final RequestProtocolCompliance requestCompliance;
104 
105     private final AsynchronousAsyncValidator asynchAsyncRevalidator;
106 
107     private final Log log = LogFactory.getLog(getClass());
108 
109     CachingHttpAsyncClient(
110             final HttpAsyncClient client,
111             final HttpCache cache,
112             final CacheConfig config) {
113         super();
114         Args.notNull(client, "HttpClient");
115         Args.notNull(cache, "HttpCache");
116         Args.notNull(config, "CacheConfig");
117         this.maxObjectSizeBytes = config.getMaxObjectSize();
118         this.sharedCache = config.isSharedCache();
119         this.backend = client;
120         this.responseCache = cache;
121         this.validityPolicy = new CacheValidityPolicy();
122         this.responseCachingPolicy = new ResponseCachingPolicy(this.maxObjectSizeBytes, this.sharedCache, false, config.is303CachingEnabled());
123         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
124         this.cacheableRequestPolicy = new CacheableRequestPolicy();
125         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, config);
126         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
127 
128         this.responseCompliance = new ResponseProtocolCompliance();
129         this.requestCompliance = new RequestProtocolCompliance(config.isWeakETagOnPutDeleteAllowed());
130 
131         this.asynchAsyncRevalidator = makeAsynchronousValidator(config);
132     }
133 
134     public CachingHttpAsyncClient() throws IOReactorException {
135         this(HttpAsyncClients.createDefault(),
136                 new BasicHttpCache(),
137                 CacheConfig.DEFAULT);
138     }
139 
140     public CachingHttpAsyncClient(final CacheConfig config) throws IOReactorException {
141         this(HttpAsyncClients.createDefault(),
142                 new BasicHttpCache(config),
143                 config);
144     }
145 
146     public CachingHttpAsyncClient(final HttpAsyncClient client) {
147         this(client,
148                 new BasicHttpCache(),
149                 CacheConfig.DEFAULT);
150     }
151 
152     public CachingHttpAsyncClient(final HttpAsyncClient client, final CacheConfig config) {
153         this(client,
154                 new BasicHttpCache(config),
155                 config);
156     }
157 
158     public CachingHttpAsyncClient(
159             final HttpAsyncClient client,
160             final ResourceFactory resourceFactory,
161             final HttpCacheStorage storage,
162             final CacheConfig config) {
163         this(client,
164                 new BasicHttpCache(resourceFactory, storage, config),
165                 config);
166     }
167 
168     public CachingHttpAsyncClient(
169             final HttpAsyncClient client,
170             final HttpCacheStorage storage,
171             final CacheConfig config) {
172         this(client,
173                 new BasicHttpCache(new HeapResourceFactory(), storage, config),
174                 config);
175     }
176 
177     CachingHttpAsyncClient(
178             final HttpAsyncClient backend,
179             final CacheValidityPolicy validityPolicy,
180             final ResponseCachingPolicy responseCachingPolicy,
181             final HttpCache responseCache,
182             final CachedHttpResponseGenerator responseGenerator,
183             final CacheableRequestPolicy cacheableRequestPolicy,
184             final CachedResponseSuitabilityChecker suitabilityChecker,
185             final ConditionalRequestBuilder conditionalRequestBuilder,
186             final ResponseProtocolCompliance responseCompliance,
187             final RequestProtocolCompliance requestCompliance) {
188         final CacheConfig config = CacheConfig.DEFAULT;
189         this.maxObjectSizeBytes = config.getMaxObjectSize();
190         this.sharedCache = config.isSharedCache();
191         this.backend = backend;
192         this.validityPolicy = validityPolicy;
193         this.responseCachingPolicy = responseCachingPolicy;
194         this.responseCache = responseCache;
195         this.responseGenerator = responseGenerator;
196         this.cacheableRequestPolicy = cacheableRequestPolicy;
197         this.suitabilityChecker = suitabilityChecker;
198         this.conditionalRequestBuilder = conditionalRequestBuilder;
199         this.responseCompliance = responseCompliance;
200         this.requestCompliance = requestCompliance;
201         this.asynchAsyncRevalidator = makeAsynchronousValidator(config);
202     }
203 
204     private AsynchronousAsyncValidator makeAsynchronousValidator(
205             final CacheConfig config) {
206         if (config.getAsynchronousWorkersMax() > 0) {
207             return new AsynchronousAsyncValidator(this, config);
208         }
209         return null;
210     }
211 
212     /**
213      * Reports the number of times that the cache successfully responded
214      * to an {@link HttpRequest} without contacting the origin server.
215      * @return the number of cache hits
216      */
217     public long getCacheHits() {
218         return this.cacheHits.get();
219     }
220 
221     /**
222      * Reports the number of times that the cache contacted the origin
223      * server because it had no appropriate response cached.
224      * @return the number of cache misses
225      */
226     public long getCacheMisses() {
227         return this.cacheMisses.get();
228     }
229 
230     /**
231      * Reports the number of times that the cache was able to satisfy
232      * a response by revalidating an existing but stale cache entry.
233      * @return the number of cache revalidations
234      */
235     public long getCacheUpdates() {
236         return this.cacheUpdates.get();
237     }
238 
239     public Future<HttpResponse> execute(
240             final HttpHost target,
241             final HttpRequest request,
242             final FutureCallback<HttpResponse> callback) {
243         return execute(target, request, null, callback);
244     }
245 
246     public <T> Future<T> execute(
247             final HttpAsyncRequestProducer requestProducer,
248             final HttpAsyncResponseConsumer<T> responseConsumer,
249             final FutureCallback<T> callback) {
250         return execute(requestProducer, responseConsumer, null, callback);
251     }
252 
253     public <T> Future<T> execute(
254             final HttpAsyncRequestProducer requestProducer,
255             final HttpAsyncResponseConsumer<T> responseConsumer,
256             final HttpContext context,
257             final FutureCallback<T> callback) {
258         this.log.warn("CachingHttpAsyncClient does not support caching for streaming HTTP exchanges");
259         return this.backend.execute(requestProducer, responseConsumer, context, callback);
260     }
261 
262     public Future<HttpResponse> execute(
263             final HttpUriRequest request,
264             final FutureCallback<HttpResponse> callback) {
265         return execute(request, null, callback);
266     }
267 
268     public Future<HttpResponse> execute(
269             final HttpUriRequest request,
270             final HttpContext context,
271             final FutureCallback<HttpResponse> callback) {
272         final URI uri = request.getURI();
273         final HttpHost httpHost = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
274         return execute(httpHost, request, context, callback);
275     }
276 
277     public Future<HttpResponse> execute(
278             final HttpHost target,
279             final HttpRequest originalRequest,
280             final HttpContext context,
281             final FutureCallback<HttpResponse> futureCallback) {
282         final BasicFuture<HttpResponse> future = new BasicFuture<HttpResponse>(futureCallback);
283         final HttpRequestWrapper request = HttpRequestWrapper.wrap(originalRequest);
284         // default response context
285         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
286 
287         final String via = generateViaHeader(request);
288 
289         if (clientRequestsOurOptions(request)) {
290             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
291             future.completed(new OptionsHttp11Response());
292             return future;
293         }
294 
295         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(
296                 request, context);
297         if (fatalErrorResponse != null) {
298             future.completed(fatalErrorResponse);
299             return future;
300         }
301 
302         try {
303             this.requestCompliance.makeRequestCompliant(request);
304         } catch (final ClientProtocolException e) {
305             future.failed(e);
306             return future;
307         }
308         request.addHeader(HeaderConstants.VIA,via);
309 
310         flushEntriesInvalidatedByRequest(target, request);
311 
312         if (!this.cacheableRequestPolicy.isServableFromCache(request)) {
313             log.debug("Request is not servable from cache");
314             callBackend(future, target, request, context);
315             return future;
316         }
317 
318         final HttpCacheEntry entry = satisfyFromCache(target, request);
319         if (entry == null) {
320             log.debug("Cache miss");
321             handleCacheMiss(future, target, request, context);
322         } else {
323             try {
324                 handleCacheHit(future, target, request, context, entry);
325             } catch (final IOException e) {
326                 future.failed(e);
327             }
328         }
329         return future;
330     }
331 
332     private void handleCacheHit(
333             final BasicFuture<HttpResponse> future,
334             final HttpHost target,
335             final HttpRequestWrapper request,
336             final HttpContext context,
337             final HttpCacheEntry entry) throws IOException {
338         recordCacheHit(target, request);
339         final HttpResponse out;
340         final Date now = getCurrentDate();
341         if (this.suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
342             log.debug("Cache hit");
343             out = generateCachedResponse(request, context, entry, now);
344         } else if (!mayCallBackend(request)) {
345             log.debug("Cache entry not suitable but only-if-cached requested");
346             out = generateGatewayTimeout(context);
347         } else if (validityPolicy.isRevalidatable(entry)
348                 && !(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
349                 && !suitabilityChecker.isConditional(request))) {
350             log.debug("Revalidating cache entry");
351             revalidateCacheEntry(future, target, request, context, entry, now);
352             return;
353         } else {
354             log.debug("Cache entry not usable; calling backend");
355             callBackend(future, target, request, context);
356             return;
357         }
358         context.setAttribute(HttpClientContext.HTTP_ROUTE, new HttpRoute(target));
359         context.setAttribute(HttpClientContext.HTTP_TARGET_HOST, target);
360         context.setAttribute(HttpClientContext.HTTP_REQUEST, request);
361         context.setAttribute(HttpClientContext.HTTP_RESPONSE, out);
362         context.setAttribute(HttpClientContext.HTTP_REQ_SENT, Boolean.TRUE);
363         future.completed(out);
364     }
365 
366     private void revalidateCacheEntry(
367             final BasicFuture<HttpResponse> future,
368             final HttpHost target,
369             final HttpRequestWrapper request,
370             final HttpContext context,
371             final HttpCacheEntry entry,
372             final Date now) throws ClientProtocolException {
373 
374         try {
375             if (this.asynchAsyncRevalidator != null
376                 && !staleResponseNotAllowed(request, entry, now)
377                 && this.validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
378                 this.log.debug("Serving stale with asynchronous revalidation");
379                 final HttpResponse resp = this.responseGenerator.generateResponse(entry);
380                 resp.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
381 
382                 this.asynchAsyncRevalidator.revalidateCacheEntry(target, request, context, entry);
383 
384                 future.completed(resp);
385                 return;
386             }
387 
388             final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
389 
390                 @Override
391                 public void failed(final Exception ex) {
392                     if(ex instanceof IOException) {
393                         super.completed(handleRevalidationFailure(request, context, entry, now));
394                     } else {
395                         super.failed(ex);
396                     }
397                 }
398 
399             };
400 
401             final BasicFuture<HttpResponse> compositeFuture = new BasicFuture<HttpResponse>(chainedFutureCallback);
402             revalidateCacheEntry(compositeFuture, target, request, context, entry);
403         } catch (final ProtocolException e) {
404             throw new ClientProtocolException(e);
405         }
406     }
407 
408     private void handleCacheMiss(
409             final BasicFuture<HttpResponse> future,
410             final HttpHost target,
411             final HttpRequestWrapper request,
412             final HttpContext context) {
413         recordCacheMiss(target, request);
414 
415         if (!mayCallBackend(request)) {
416             future.completed(new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
417             return;
418         }
419 
420         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
421         if (variants != null && variants.size() > 0) {
422             negotiateResponseFromVariants(future, target, request, context, variants);
423             return;
424         }
425 
426         callBackend(future, target, request, context);
427     }
428 
429     private HttpCacheEntry satisfyFromCache(
430             final HttpHost target,
431             final HttpRequest request) {
432         HttpCacheEntry entry = null;
433         try {
434             entry = this.responseCache.getCacheEntry(target, request);
435         } catch (final IOException ioe) {
436             this.log.warn("Unable to retrieve entries from cache", ioe);
437         }
438         return entry;
439     }
440 
441     private HttpResponse getFatallyNoncompliantResponse(
442             final HttpRequest request,
443             final HttpContext context) {
444         HttpResponse fatalErrorResponse = null;
445         final List<RequestProtocolError> fatalError = this.requestCompliance.requestIsFatallyNonCompliant(request);
446 
447         for (final RequestProtocolError error : fatalError) {
448             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
449             fatalErrorResponse = this.requestCompliance.getErrorForRequest(error);
450         }
451         return fatalErrorResponse;
452     }
453 
454     private Map<String, Variant> getExistingCacheVariants(
455             final HttpHost target,
456             final HttpRequest request) {
457         Map<String,Variant> variants = null;
458         try {
459             variants = this.responseCache.getVariantCacheEntriesWithEtags(target, request);
460         } catch (final IOException ioe) {
461             this.log.warn("Unable to retrieve variant entries from cache", ioe);
462         }
463         return variants;
464     }
465 
466     private void recordCacheMiss(final HttpHost target, final HttpRequest request) {
467         this.cacheMisses.getAndIncrement();
468         if (this.log.isDebugEnabled()) {
469             final RequestLine rl = request.getRequestLine();
470             this.log.debug("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
471         }
472     }
473 
474     private void recordCacheHit(final HttpHost target, final HttpRequest request) {
475         this.cacheHits.getAndIncrement();
476         if (this.log.isDebugEnabled()) {
477             final RequestLine rl = request.getRequestLine();
478             this.log.debug("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
479         }
480     }
481 
482     private void recordCacheUpdate(final HttpContext context) {
483         this.cacheUpdates.getAndIncrement();
484         setResponseStatus(context, CacheResponseStatus.VALIDATED);
485     }
486 
487     private void flushEntriesInvalidatedByRequest(final HttpHost target,
488             final HttpRequest request) {
489         try {
490             this.responseCache.flushInvalidatedCacheEntriesFor(target, request);
491         } catch (final IOException ioe) {
492             this.log.warn("Unable to flush invalidated entries from cache", ioe);
493         }
494     }
495 
496     private HttpResponse generateCachedResponse(
497             final HttpRequest request,
498             final HttpContext context,
499             final HttpCacheEntry entry,
500             final Date now) {
501         final HttpResponse cachedResponse;
502         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
503                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
504             cachedResponse = this.responseGenerator.generateNotModifiedResponse(entry);
505         } else {
506             cachedResponse = this.responseGenerator.generateResponse(entry);
507         }
508         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
509         if (this.validityPolicy.getStalenessSecs(entry, now) > 0L) {
510             cachedResponse.addHeader("Warning","110 localhost \"Response is stale\"");
511         }
512         return cachedResponse;
513     }
514 
515     private HttpResponse handleRevalidationFailure(
516             final HttpRequest request,
517             final HttpContext context,
518             final HttpCacheEntry entry,
519             final Date now) {
520         if (staleResponseNotAllowed(request, entry, now)) {
521             return generateGatewayTimeout(context);
522         }
523         return unvalidatedCacheHit(context, entry);
524     }
525 
526     private HttpResponse generateGatewayTimeout(final HttpContext context) {
527         setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
528         return new BasicHttpResponse(HttpVersion.HTTP_1_1,
529                 HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout");
530     }
531 
532     private HttpResponse unvalidatedCacheHit(
533             final HttpContext context,
534             final HttpCacheEntry entry) {
535         final HttpResponse cachedResponse = this.responseGenerator.generateResponse(entry);
536         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
537         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
538         return cachedResponse;
539     }
540 
541     private boolean staleResponseNotAllowed(
542             final HttpRequest request,
543             final HttpCacheEntry entry,
544             final Date now) {
545         return this.validityPolicy.mustRevalidate(entry)
546             || (isSharedCache() && this.validityPolicy.proxyRevalidate(entry))
547             || explicitFreshnessRequest(request, entry, now);
548     }
549 
550     private boolean mayCallBackend(final HttpRequest request) {
551         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
552             for (final HeaderElement elt : h.getElements()) {
553                 if ("only-if-cached".equals(elt.getName())) {
554                     this.log.debug("Request marked only-if-cached");
555                     return false;
556                 }
557             }
558         }
559         return true;
560     }
561 
562     private boolean explicitFreshnessRequest(
563             final HttpRequest request,
564             final HttpCacheEntry entry,
565             final Date now) {
566         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
567             for(final HeaderElement elt : h.getElements()) {
568                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
569                     try {
570                         final int maxstale = Integer.parseInt(elt.getValue());
571                         final long age = this.validityPolicy.getCurrentAgeSecs(entry, now);
572                         final long lifetime = this.validityPolicy.getFreshnessLifetimeSecs(entry);
573                         if (age - lifetime > maxstale) {
574                             return true;
575                         }
576                     } catch (final NumberFormatException nfe) {
577                         return true;
578                     }
579                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
580                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
581                     return true;
582                 }
583             }
584         }
585         return false;
586     }
587 
588     private String generateViaHeader(final HttpMessage msg) {
589 
590         final ProtocolVersion pv = msg.getProtocolVersion();
591         final String existingEntry = viaHeaders.get(pv);
592         if (existingEntry != null) {
593             return existingEntry;
594         }
595 
596         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
597         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
598 
599         final String value;
600         if ("http".equalsIgnoreCase(pv.getProtocol())) {
601             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getMajor(), pv.getMinor(),
602                     release);
603         } else {
604             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), pv.getMajor(),
605                     pv.getMinor(), release);
606         }
607         viaHeaders.put(pv, value);
608 
609         return value;
610     }
611 
612     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
613         if (context != null) {
614             context.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
615         }
616     }
617 
618     /**
619      * Reports whether this {@code CachingHttpClient} implementation
620      * supports byte-range requests as specified by the {@code Range}
621      * and {@code Content-Range} headers.
622      * @return {@code true} if byte-range requests are supported
623      */
624     public boolean supportsRangeAndContentRangeHeaders() {
625         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
626     }
627 
628     /**
629      * Reports whether this {@code CachingHttpClient} is configured as
630      * a shared (public) or non-shared (private) cache. See {@link
631      * CacheConfig#setSharedCache(boolean)}.
632      * @return {@code true} if we are behaving as a shared (public)
633      *   cache
634      */
635     public boolean isSharedCache() {
636         return this.sharedCache;
637     }
638 
639     Date getCurrentDate() {
640         return new Date();
641     }
642 
643     boolean clientRequestsOurOptions(final HttpRequest request) {
644         final RequestLine line = request.getRequestLine();
645 
646         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
647             return false;
648         }
649 
650         if (!"*".equals(line.getUri())) {
651             return false;
652         }
653         return "0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue());
654     }
655 
656     void callBackend(
657             final BasicFuture<HttpResponse> future,
658             final HttpHost target,
659             final HttpRequestWrapper request,
660             final HttpContext context) {
661         final Date requestDate = getCurrentDate();
662         this.log.trace("Calling the backend");
663 
664         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
665 
666             @Override
667             public void completed(final HttpResponse httpResponse) {
668                 httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
669                 try {
670                     final HttpResponse backendResponse = handleBackendResponse(target, request, requestDate, getCurrentDate(), httpResponse);
671                     super.completed(backendResponse);
672                 } catch (final IOException e) {
673                     super.failed(e);
674                 }
675 
676             }
677 
678         };
679         this.backend.execute(target, request, context, chainedFutureCallback);
680     }
681 
682     private boolean revalidationResponseIsTooOld(
683             final HttpResponse backendResponse,
684             final HttpCacheEntry cacheEntry) {
685         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
686         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
687         if (entryDateHeader != null && responseDateHeader != null) {
688             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
689             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
690             if (respDate != null && respDate.before(entryDate)) {
691                 return true;
692             }
693         }
694         return false;
695     }
696 
697     void negotiateResponseFromVariants(
698             final BasicFuture<HttpResponse> future,
699             final HttpHost target,
700             final HttpRequestWrapper request,
701             final HttpContext context,
702             final Map<String, Variant> variants) {
703         final HttpRequest conditionalRequest = this.conditionalRequestBuilder.buildConditionalRequestFromVariants(request, variants);
704 
705         final Date requestDate = getCurrentDate();
706 
707         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
708 
709             @Override
710             public void completed(final HttpResponse httpResponse) {
711                 final Date responseDate = getCurrentDate();
712 
713                 httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
714 
715                 if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
716                     try {
717                         future.completed(handleBackendResponse(target, request, requestDate, responseDate, httpResponse));
718                         return;
719                     } catch (final IOException e) {
720                         future.failed(e);
721                         return;
722                     }
723                 }
724 
725                 final Header resultEtagHeader = httpResponse.getFirstHeader(HeaderConstants.ETAG);
726                 if (resultEtagHeader == null) {
727                     CachingHttpAsyncClient.this.log.warn("304 response did not contain ETag");
728                     callBackend(future, target, request, context);
729                     return;
730                 }
731 
732                 final String resultEtag = resultEtagHeader.getValue();
733                 final Variant matchingVariant = variants.get(resultEtag);
734                 if (matchingVariant == null) {
735                     CachingHttpAsyncClient.this.log.debug("304 response did not contain ETag matching one sent in If-None-Match");
736                     callBackend(future, target, request, context);
737                 }
738 
739                 final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
740 
741                 if (revalidationResponseIsTooOld(httpResponse, matchedEntry)) {
742                     EntityUtils.consumeQuietly(httpResponse.getEntity());
743                     retryRequestUnconditionally(future, target, request, context, matchedEntry);
744                     return;
745                 }
746 
747                 recordCacheUpdate(context);
748 
749                 final HttpCacheEntry responseEntry = getUpdatedVariantEntry(target,
750                         conditionalRequest, requestDate, responseDate, httpResponse,
751                         matchingVariant, matchedEntry);
752 
753                 final HttpResponse resp = CachingHttpAsyncClient.this.responseGenerator.generateResponse(responseEntry);
754                 tryToUpdateVariantMap(target, request, matchingVariant);
755 
756                 if (shouldSendNotModifiedResponse(request, responseEntry)) {
757                     future.completed(CachingHttpAsyncClient.this.responseGenerator.generateNotModifiedResponse(responseEntry));
758                     return;
759                 }
760 
761                 future.completed(resp);
762             }
763 
764         };
765 
766         this.backend.execute(target, conditionalRequest, context, chainedFutureCallback);
767     }
768 
769     private void retryRequestUnconditionally(
770             final BasicFuture<HttpResponse> future,
771             final HttpHost target,
772             final HttpRequestWrapper request,
773             final HttpContext context,
774             final HttpCacheEntry matchedEntry) {
775         final HttpRequestWrapper unconditional = this.conditionalRequestBuilder
776             .buildUnconditionalRequest(request, matchedEntry);
777         callBackend(future, target, unconditional, context);
778     }
779 
780     private HttpCacheEntry getUpdatedVariantEntry(
781             final HttpHost target,
782             final HttpRequest conditionalRequest,
783             final Date requestDate,
784             final Date responseDate,
785             final HttpResponse backendResponse,
786             final Variant matchingVariant,
787             final HttpCacheEntry matchedEntry) {
788         HttpCacheEntry responseEntry = matchedEntry;
789         try {
790             responseEntry = this.responseCache.updateVariantCacheEntry(target, conditionalRequest,
791                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
792         } catch (final IOException ioe) {
793             this.log.warn("Could not update cache entry", ioe);
794         }
795         return responseEntry;
796     }
797 
798     private void tryToUpdateVariantMap(
799             final HttpHost target,
800             final HttpRequest request,
801             final Variant matchingVariant) {
802         try {
803             this.responseCache.reuseVariantEntryFor(target, request, matchingVariant);
804         } catch (final IOException ioe) {
805             this.log.warn("Could not update cache entry to reuse variant", ioe);
806         }
807     }
808 
809     private boolean shouldSendNotModifiedResponse(
810             final HttpRequest request,
811             final HttpCacheEntry responseEntry) {
812         return (this.suitabilityChecker.isConditional(request)
813                 && this.suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
814     }
815 
816     void revalidateCacheEntry(
817             final BasicFuture<HttpResponse> future,
818             final HttpHost target,
819             final HttpRequestWrapper request,
820             final HttpContext context,
821             final HttpCacheEntry cacheEntry) throws ProtocolException {
822 
823         final HttpRequestWrapper conditionalRequest = this.conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
824         final Date requestDate = getCurrentDate();
825 
826         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
827 
828             @Override
829             public void completed(final HttpResponse httpResponse) {
830                 final Date responseDate = getCurrentDate();
831 
832                 if (revalidationResponseIsTooOld(httpResponse, cacheEntry)) {
833                     final HttpRequest unconditional = CachingHttpAsyncClient.this.conditionalRequestBuilder.buildUnconditionalRequest(request, cacheEntry);
834                     final Date innerRequestDate = getCurrentDate();
835 
836                     final ChainedFutureCallback<HttpResponse> chainedFutureCallback2 = new ChainedFutureCallback<HttpResponse>(future) {
837 
838                         @Override
839                         public void completed(final HttpResponse innerHttpResponse) {
840                             final Date innerResponseDate = getCurrentDate();
841                             revalidateCacheEntryCompleted(future,
842                                     target, request, context, cacheEntry,
843                                     conditionalRequest, innerRequestDate,
844                                     innerHttpResponse, innerResponseDate);
845                         }
846 
847                     };
848                     CachingHttpAsyncClient.this.backend.execute(target, unconditional, context, chainedFutureCallback2);
849                 }
850 
851                 revalidateCacheEntryCompleted(future,
852                         target, request, context, cacheEntry,
853                         conditionalRequest, requestDate,
854                         httpResponse, responseDate);
855             }
856 
857         };
858 
859         this.backend.execute(target, conditionalRequest, context, chainedFutureCallback);
860     }
861 
862     private void revalidateCacheEntryCompleted(
863             final BasicFuture<HttpResponse> future,
864             final HttpHost target,
865             final HttpRequestWrapper request,
866             final HttpContext context,
867             final HttpCacheEntry cacheEntry,
868             final HttpRequestWrapper conditionalRequest,
869             final Date requestDate,
870             final HttpResponse httpResponse,
871             final Date responseDate) {
872 
873         httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
874 
875         final int statusCode = httpResponse.getStatusLine().getStatusCode();
876         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
877             recordCacheUpdate(context);
878         }
879 
880         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
881             final HttpCacheEntry updatedEntry;
882             try {
883                 updatedEntry = CachingHttpAsyncClient.this.responseCache.updateCacheEntry(target, request, cacheEntry,
884                         httpResponse, requestDate, responseDate);
885             } catch (final IOException e) {
886                 future.failed(e);
887                 return;
888             }
889             if (CachingHttpAsyncClient.this.suitabilityChecker.isConditional(request)
890                     && CachingHttpAsyncClient.this.suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
891                 future.completed(CachingHttpAsyncClient.this.responseGenerator.generateNotModifiedResponse(updatedEntry));
892                 return;
893             }
894             future.completed(CachingHttpAsyncClient.this.responseGenerator.generateResponse(updatedEntry));
895             return;
896         }
897 
898         if (staleIfErrorAppliesTo(statusCode)
899             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
900             && CachingHttpAsyncClient.this.validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
901             final HttpResponse cachedResponse = CachingHttpAsyncClient.this.responseGenerator.generateResponse(cacheEntry);
902             cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
903             future.completed(cachedResponse);
904             return;
905         }
906 
907         try {
908             final HttpResponse backendResponse = handleBackendResponse(target, conditionalRequest,
909                     requestDate, responseDate, httpResponse);
910             future.completed(backendResponse);
911         } catch (final IOException e) {
912             future.failed(e);
913         }
914     }
915 
916     private boolean staleIfErrorAppliesTo(final int statusCode) {
917         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
918                 || statusCode == HttpStatus.SC_BAD_GATEWAY
919                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
920                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
921     }
922 
923     HttpResponse handleBackendResponse(
924             final HttpHost target,
925             final HttpRequestWrapper request,
926             final Date requestDate,
927             final Date responseDate,
928             final HttpResponse backendResponse) throws IOException {
929 
930         this.log.debug("Handling Backend response");
931         this.responseCompliance.ensureProtocolCompliance(request, backendResponse);
932 
933         final boolean cacheable = this.responseCachingPolicy.isResponseCacheable(request, backendResponse);
934         this.responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
935         if (cacheable &&
936             !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
937             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
938             return this.responseCache.cacheAndReturnResponse(target, request, backendResponse, requestDate,
939                     responseDate);
940         }
941         if (!cacheable) {
942             try {
943                 this.responseCache.flushCacheEntriesFor(target, request);
944             } catch (final IOException ioe) {
945                 this.log.warn("Unable to flush invalid cache entries", ioe);
946             }
947         }
948         return backendResponse;
949     }
950 
951     /**
952      * For 304 Not modified responses, adds a "Last-Modified" header with the
953      * value of the "If-Modified-Since" header passed in the request. This
954      * header is required to be able to reuse match the cache entry for
955      * subsequent requests but as defined in http specifications it is not
956      * included in 304 responses by backend servers. This header will not be
957      * included in the resulting response.
958      */
959     private void storeRequestIfModifiedSinceFor304Response(
960             final HttpRequest request,
961             final HttpResponse backendResponse) {
962         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
963             final Header h = request.getFirstHeader("If-Modified-Since");
964             if (h != null) {
965                 backendResponse.addHeader("Last-Modified", h.getValue());
966             }
967         }
968     }
969 
970     private boolean alreadyHaveNewerCacheEntry(
971             final HttpHost target,
972             final HttpRequest request,
973             final HttpResponse backendResponse) {
974         HttpCacheEntry existing = null;
975         try {
976             existing = this.responseCache.getCacheEntry(target, request);
977         } catch (final IOException ioe) {
978             // nop
979         }
980         if (existing == null) {
981             return false;
982         }
983         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
984         if (entryDateHeader == null) {
985             return false;
986         }
987         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
988         if (responseDateHeader == null) {
989             return false;
990         }
991         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
992         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
993         return responseDate != null && responseDate.before(entryDate);
994     }
995 
996 }