View Javadoc

1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.net.URI;
31  import java.util.Date;
32  import java.util.HashMap;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.concurrent.Future;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.http.Header;
41  import org.apache.http.HeaderElement;
42  import org.apache.http.HttpHost;
43  import org.apache.http.HttpMessage;
44  import org.apache.http.HttpRequest;
45  import org.apache.http.HttpResponse;
46  import org.apache.http.HttpStatus;
47  import org.apache.http.HttpVersion;
48  import org.apache.http.ProtocolException;
49  import org.apache.http.ProtocolVersion;
50  import org.apache.http.RequestLine;
51  import org.apache.http.annotation.ThreadSafe;
52  import org.apache.http.client.ClientProtocolException;
53  import org.apache.http.client.cache.CacheResponseStatus;
54  import org.apache.http.client.cache.HeaderConstants;
55  import org.apache.http.client.cache.HttpCacheContext;
56  import org.apache.http.client.cache.HttpCacheEntry;
57  import org.apache.http.client.cache.HttpCacheStorage;
58  import org.apache.http.client.cache.ResourceFactory;
59  import org.apache.http.client.methods.CloseableHttpResponse;
60  import org.apache.http.client.methods.HttpRequestWrapper;
61  import org.apache.http.client.methods.HttpUriRequest;
62  import org.apache.http.client.protocol.HttpClientContext;
63  import org.apache.http.client.utils.DateUtils;
64  import org.apache.http.concurrent.BasicFuture;
65  import org.apache.http.concurrent.FutureCallback;
66  import org.apache.http.conn.routing.HttpRoute;
67  import org.apache.http.impl.nio.client.HttpAsyncClients;
68  import org.apache.http.message.BasicHttpResponse;
69  import org.apache.http.nio.client.HttpAsyncClient;
70  import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
71  import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
72  import org.apache.http.nio.reactor.IOReactorException;
73  import org.apache.http.protocol.HTTP;
74  import org.apache.http.protocol.HttpContext;
75  import org.apache.http.util.Args;
76  import org.apache.http.util.EntityUtils;
77  import org.apache.http.util.VersionInfo;
78  
79  @ThreadSafe // So long as the responseCache implementation is threadsafe
80  public class CachingHttpAsyncClient implements HttpAsyncClient {
81  
82      private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
83  
84      private final AtomicLong cacheHits = new AtomicLong();
85      private final AtomicLong cacheMisses = new AtomicLong();
86      private final AtomicLong cacheUpdates = new AtomicLong();
87  
88      private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
89  
90      private final HttpAsyncClient backend;
91      private final HttpCache responseCache;
92      private final CacheValidityPolicy validityPolicy;
93      private final ResponseCachingPolicy responseCachingPolicy;
94      private final CachedHttpResponseGenerator responseGenerator;
95      private final CacheableRequestPolicy cacheableRequestPolicy;
96      private final CachedResponseSuitabilityChecker suitabilityChecker;
97  
98      private final ConditionalRequestBuilder conditionalRequestBuilder;
99  
100     private final long maxObjectSizeBytes;
101     private final boolean sharedCache;
102 
103     private final ResponseProtocolCompliance responseCompliance;
104     private final RequestProtocolCompliance requestCompliance;
105 
106     private final AsynchronousAsyncValidator asynchAsyncRevalidator;
107 
108     private final Log log = LogFactory.getLog(getClass());
109 
110     CachingHttpAsyncClient(
111             final HttpAsyncClient client,
112             final HttpCache cache,
113             final CacheConfig config) {
114         super();
115         Args.notNull(client, "HttpClient");
116         Args.notNull(cache, "HttpCache");
117         Args.notNull(config, "CacheConfig");
118         this.maxObjectSizeBytes = config.getMaxObjectSize();
119         this.sharedCache = config.isSharedCache();
120         this.backend = client;
121         this.responseCache = cache;
122         this.validityPolicy = new CacheValidityPolicy();
123         this.responseCachingPolicy = new ResponseCachingPolicy(this.maxObjectSizeBytes, this.sharedCache, false, config.is303CachingEnabled());
124         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
125         this.cacheableRequestPolicy = new CacheableRequestPolicy();
126         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, config);
127         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
128 
129         this.responseCompliance = new ResponseProtocolCompliance();
130         this.requestCompliance = new RequestProtocolCompliance(config.isWeakETagOnPutDeleteAllowed());
131 
132         this.asynchAsyncRevalidator = makeAsynchronousValidator(config);
133     }
134 
135     public CachingHttpAsyncClient() throws IOReactorException {
136         this(HttpAsyncClients.createDefault(),
137                 new BasicHttpCache(),
138                 CacheConfig.DEFAULT);
139     }
140 
141     public CachingHttpAsyncClient(final CacheConfig config) throws IOReactorException {
142         this(HttpAsyncClients.createDefault(),
143                 new BasicHttpCache(config),
144                 config);
145     }
146 
147     public CachingHttpAsyncClient(final HttpAsyncClient client) {
148         this(client,
149                 new BasicHttpCache(),
150                 CacheConfig.DEFAULT);
151     }
152 
153     public CachingHttpAsyncClient(final HttpAsyncClient client, final CacheConfig config) {
154         this(client,
155                 new BasicHttpCache(config),
156                 config);
157     }
158 
159     public CachingHttpAsyncClient(
160             final HttpAsyncClient client,
161             final ResourceFactory resourceFactory,
162             final HttpCacheStorage storage,
163             final CacheConfig config) {
164         this(client,
165                 new BasicHttpCache(resourceFactory, storage, config),
166                 config);
167     }
168 
169     public CachingHttpAsyncClient(
170             final HttpAsyncClient client,
171             final HttpCacheStorage storage,
172             final CacheConfig config) {
173         this(client,
174                 new BasicHttpCache(new HeapResourceFactory(), storage, config),
175                 config);
176     }
177 
178     CachingHttpAsyncClient(
179             final HttpAsyncClient backend,
180             final CacheValidityPolicy validityPolicy,
181             final ResponseCachingPolicy responseCachingPolicy,
182             final HttpCache responseCache,
183             final CachedHttpResponseGenerator responseGenerator,
184             final CacheableRequestPolicy cacheableRequestPolicy,
185             final CachedResponseSuitabilityChecker suitabilityChecker,
186             final ConditionalRequestBuilder conditionalRequestBuilder,
187             final ResponseProtocolCompliance responseCompliance,
188             final RequestProtocolCompliance requestCompliance) {
189         final CacheConfig config = CacheConfig.DEFAULT;
190         this.maxObjectSizeBytes = config.getMaxObjectSize();
191         this.sharedCache = config.isSharedCache();
192         this.backend = backend;
193         this.validityPolicy = validityPolicy;
194         this.responseCachingPolicy = responseCachingPolicy;
195         this.responseCache = responseCache;
196         this.responseGenerator = responseGenerator;
197         this.cacheableRequestPolicy = cacheableRequestPolicy;
198         this.suitabilityChecker = suitabilityChecker;
199         this.conditionalRequestBuilder = conditionalRequestBuilder;
200         this.responseCompliance = responseCompliance;
201         this.requestCompliance = requestCompliance;
202         this.asynchAsyncRevalidator = makeAsynchronousValidator(config);
203     }
204 
205     private AsynchronousAsyncValidator makeAsynchronousValidator(
206             final CacheConfig config) {
207         if (config.getAsynchronousWorkersMax() > 0) {
208             return new AsynchronousAsyncValidator(this, config);
209         }
210         return null;
211     }
212 
213     /**
214      * Reports the number of times that the cache successfully responded
215      * to an {@link HttpRequest} without contacting the origin server.
216      * @return the number of cache hits
217      */
218     public long getCacheHits() {
219         return this.cacheHits.get();
220     }
221 
222     /**
223      * Reports the number of times that the cache contacted the origin
224      * server because it had no appropriate response cached.
225      * @return the number of cache misses
226      */
227     public long getCacheMisses() {
228         return this.cacheMisses.get();
229     }
230 
231     /**
232      * Reports the number of times that the cache was able to satisfy
233      * a response by revalidating an existing but stale cache entry.
234      * @return the number of cache revalidations
235      */
236     public long getCacheUpdates() {
237         return this.cacheUpdates.get();
238     }
239 
240     public Future<HttpResponse> execute(
241             final HttpHost target,
242             final HttpRequest request,
243             final FutureCallback<HttpResponse> callback) {
244         return execute(target, request, null, callback);
245     }
246 
247     public <T> Future<T> execute(
248             final HttpAsyncRequestProducer requestProducer,
249             final HttpAsyncResponseConsumer<T> responseConsumer,
250             final FutureCallback<T> callback) {
251         return execute(requestProducer, responseConsumer, null, callback);
252     }
253 
254     public <T> Future<T> execute(
255             final HttpAsyncRequestProducer requestProducer,
256             final HttpAsyncResponseConsumer<T> responseConsumer,
257             final HttpContext context,
258             final FutureCallback<T> callback) {
259         this.log.warn("CachingHttpAsyncClient does not support caching for streaming HTTP exchanges");
260         return this.backend.execute(requestProducer, responseConsumer, context, callback);
261     }
262 
263     public Future<HttpResponse> execute(
264             final HttpUriRequest request,
265             final FutureCallback<HttpResponse> callback) {
266         return execute(request, null, callback);
267     }
268 
269     public Future<HttpResponse> execute(
270             final HttpUriRequest request,
271             final HttpContext context,
272             final FutureCallback<HttpResponse> callback) {
273         final URI uri = request.getURI();
274         final HttpHost httpHost = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
275         return execute(httpHost, request, context, callback);
276     }
277 
278     public Future<HttpResponse> execute(
279             final HttpHost target,
280             final HttpRequest originalRequest,
281             final HttpContext context,
282             final FutureCallback<HttpResponse> futureCallback) {
283         final BasicFuture<HttpResponse> future = new BasicFuture<HttpResponse>(futureCallback);
284         final HttpRequestWrapper request = HttpRequestWrapper.wrap(originalRequest);
285         final HttpCacheContext clientContext = HttpCacheContext.adapt(context);
286         // default response context
287         setResponseStatus(clientContext, CacheResponseStatus.CACHE_MISS);
288 
289         final String via = generateViaHeader(request);
290 
291         if (clientRequestsOurOptions(request)) {
292             setResponseStatus(clientContext, CacheResponseStatus.CACHE_MODULE_RESPONSE);
293             future.completed(new OptionsHttp11Response());
294             return future;
295         }
296 
297         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(
298                 request, clientContext);
299         if (fatalErrorResponse != null) {
300             future.completed(fatalErrorResponse);
301             return future;
302         }
303 
304         try {
305             this.requestCompliance.makeRequestCompliant(request);
306         } catch (final ClientProtocolException e) {
307             future.failed(e);
308             return future;
309         }
310         request.addHeader(HeaderConstants.VIA,via);
311 
312         flushEntriesInvalidatedByRequest(target, request);
313 
314         if (!this.cacheableRequestPolicy.isServableFromCache(request)) {
315             log.debug("Request is not servable from cache");
316             callBackend(future, target, request, clientContext);
317             return future;
318         }
319 
320         final HttpCacheEntry entry = satisfyFromCache(target, request);
321         if (entry == null) {
322             log.debug("Cache miss");
323             handleCacheMiss(future, target, request, clientContext);
324         } else {
325             try {
326                 handleCacheHit(future, target, request, clientContext, entry);
327             } catch (final IOException e) {
328                 future.failed(e);
329             }
330         }
331         return future;
332     }
333 
334     private void handleCacheHit(
335             final BasicFuture<HttpResponse> future,
336             final HttpHost target,
337             final HttpRequestWrapper request,
338             final HttpCacheContext clientContext,
339             final HttpCacheEntry entry) throws IOException {
340         recordCacheHit(target, request);
341         final HttpResponse out;
342         final Date now = getCurrentDate();
343         if (this.suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
344             log.debug("Cache hit");
345             out = generateCachedResponse(request, clientContext, entry, now);
346         } else if (!mayCallBackend(request)) {
347             log.debug("Cache entry not suitable but only-if-cached requested");
348             out = generateGatewayTimeout(clientContext);
349         } else if (validityPolicy.isRevalidatable(entry)
350                 && !(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
351                 && !suitabilityChecker.isConditional(request))) {
352             log.debug("Revalidating cache entry");
353             revalidateCacheEntry(future, target, request, clientContext, entry, now);
354             return;
355         } else {
356             log.debug("Cache entry not usable; calling backend");
357             callBackend(future, target, request, clientContext);
358             return;
359         }
360         clientContext.setAttribute(HttpClientContext.HTTP_ROUTE, new HttpRoute(target));
361         clientContext.setAttribute(HttpClientContext.HTTP_TARGET_HOST, target);
362         clientContext.setAttribute(HttpClientContext.HTTP_REQUEST, request);
363         clientContext.setAttribute(HttpClientContext.HTTP_RESPONSE, out);
364         clientContext.setAttribute(HttpClientContext.HTTP_REQ_SENT, Boolean.TRUE);
365         future.completed(out);
366     }
367 
368     private void revalidateCacheEntry(
369             final BasicFuture<HttpResponse> future,
370             final HttpHost target,
371             final HttpRequestWrapper request,
372             final HttpCacheContext clientContext,
373             final HttpCacheEntry entry,
374             final Date now) throws ClientProtocolException {
375 
376         try {
377             if (this.asynchAsyncRevalidator != null
378                 && !staleResponseNotAllowed(request, entry, now)
379                 && this.validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
380                 this.log.debug("Serving stale with asynchronous revalidation");
381                 final HttpResponse resp = this.responseGenerator.generateResponse(entry);
382                 resp.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
383 
384                 this.asynchAsyncRevalidator.revalidateCacheEntry(target, request, clientContext, entry);
385 
386                 future.completed(resp);
387                 return;
388             }
389 
390             final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
391 
392                 @Override
393                 public void failed(final Exception ex) {
394                     if(ex instanceof IOException) {
395                         super.completed(handleRevalidationFailure(request, clientContext, entry, now));
396                     } else {
397                         super.failed(ex);
398                     }
399                 }
400 
401             };
402 
403             final BasicFuture<HttpResponse> compositeFuture = new BasicFuture<HttpResponse>(chainedFutureCallback);
404             revalidateCacheEntry(compositeFuture, target, request, clientContext, entry);
405         } catch (final ProtocolException e) {
406             throw new ClientProtocolException(e);
407         }
408     }
409 
410     private void handleCacheMiss(
411             final BasicFuture<HttpResponse> future,
412             final HttpHost target,
413             final HttpRequestWrapper request,
414             final HttpCacheContext clientContext) {
415         recordCacheMiss(target, request);
416 
417         if (!mayCallBackend(request)) {
418             future.completed(new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
419             return;
420         }
421 
422         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
423         if (variants != null && variants.size() > 0) {
424             negotiateResponseFromVariants(future, target, request, clientContext, variants);
425             return;
426         }
427 
428         callBackend(future, target, request, clientContext);
429     }
430 
431     private HttpCacheEntry satisfyFromCache(
432             final HttpHost target,
433             final HttpRequest request) {
434         HttpCacheEntry entry = null;
435         try {
436             entry = this.responseCache.getCacheEntry(target, request);
437         } catch (final IOException ioe) {
438             this.log.warn("Unable to retrieve entries from cache", ioe);
439         }
440         return entry;
441     }
442 
443     private HttpResponse getFatallyNoncompliantResponse(
444             final HttpRequest request,
445             final HttpCacheContext clientContext) {
446         HttpResponse fatalErrorResponse = null;
447         final List<RequestProtocolError> fatalError = this.requestCompliance.requestIsFatallyNonCompliant(request);
448 
449         for (final RequestProtocolError error : fatalError) {
450             setResponseStatus(clientContext, CacheResponseStatus.CACHE_MODULE_RESPONSE);
451             fatalErrorResponse = this.requestCompliance.getErrorForRequest(error);
452         }
453         return fatalErrorResponse;
454     }
455 
456     private Map<String, Variant> getExistingCacheVariants(
457             final HttpHost target,
458             final HttpRequest request) {
459         Map<String,Variant> variants = null;
460         try {
461             variants = this.responseCache.getVariantCacheEntriesWithEtags(target, request);
462         } catch (final IOException ioe) {
463             this.log.warn("Unable to retrieve variant entries from cache", ioe);
464         }
465         return variants;
466     }
467 
468     private void recordCacheMiss(final HttpHost target, final HttpRequest request) {
469         this.cacheMisses.getAndIncrement();
470         if (this.log.isDebugEnabled()) {
471             final RequestLine rl = request.getRequestLine();
472             this.log.debug("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
473         }
474     }
475 
476     private void recordCacheHit(final HttpHost target, final HttpRequest request) {
477         this.cacheHits.getAndIncrement();
478         if (this.log.isDebugEnabled()) {
479             final RequestLine rl = request.getRequestLine();
480             this.log.debug("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
481         }
482     }
483 
484     private void recordCacheUpdate(final HttpCacheContext clientContext) {
485         this.cacheUpdates.getAndIncrement();
486         setResponseStatus(clientContext, CacheResponseStatus.VALIDATED);
487     }
488 
489     private void flushEntriesInvalidatedByRequest(final HttpHost target,
490             final HttpRequest request) {
491         try {
492             this.responseCache.flushInvalidatedCacheEntriesFor(target, request);
493         } catch (final IOException ioe) {
494             this.log.warn("Unable to flush invalidated entries from cache", ioe);
495         }
496     }
497 
498     private HttpResponse generateCachedResponse(
499             final HttpRequest request,
500             final HttpCacheContext clientContext,
501             final HttpCacheEntry entry,
502             final Date now) {
503         final HttpResponse cachedResponse;
504         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
505                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
506             cachedResponse = this.responseGenerator.generateNotModifiedResponse(entry);
507         } else {
508             cachedResponse = this.responseGenerator.generateResponse(entry);
509         }
510         setResponseStatus(clientContext, CacheResponseStatus.CACHE_HIT);
511         if (this.validityPolicy.getStalenessSecs(entry, now) > 0L) {
512             cachedResponse.addHeader("Warning","110 localhost \"Response is stale\"");
513         }
514         return cachedResponse;
515     }
516 
517     private HttpResponse handleRevalidationFailure(
518             final HttpRequest request,
519             final HttpCacheContext clientContext,
520             final HttpCacheEntry entry,
521             final Date now) {
522         if (staleResponseNotAllowed(request, entry, now)) {
523             return generateGatewayTimeout(clientContext);
524         }
525         return unvalidatedCacheHit(clientContext, entry);
526     }
527 
528     private HttpResponse generateGatewayTimeout(final HttpCacheContext clientContext) {
529         setResponseStatus(clientContext, CacheResponseStatus.CACHE_MODULE_RESPONSE);
530         return new BasicHttpResponse(HttpVersion.HTTP_1_1,
531                 HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout");
532     }
533 
534     private HttpResponse unvalidatedCacheHit(
535             final HttpCacheContext clientContext,
536             final HttpCacheEntry entry) {
537         final HttpResponse cachedResponse = this.responseGenerator.generateResponse(entry);
538         setResponseStatus(clientContext, CacheResponseStatus.CACHE_HIT);
539         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
540         return cachedResponse;
541     }
542 
543     private boolean staleResponseNotAllowed(
544             final HttpRequest request,
545             final HttpCacheEntry entry,
546             final Date now) {
547         return this.validityPolicy.mustRevalidate(entry)
548             || (isSharedCache() && this.validityPolicy.proxyRevalidate(entry))
549             || explicitFreshnessRequest(request, entry, now);
550     }
551 
552     private boolean mayCallBackend(final HttpRequest request) {
553         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
554             for (final HeaderElement elt : h.getElements()) {
555                 if ("only-if-cached".equals(elt.getName())) {
556                     this.log.debug("Request marked only-if-cached");
557                     return false;
558                 }
559             }
560         }
561         return true;
562     }
563 
564     private boolean explicitFreshnessRequest(
565             final HttpRequest request,
566             final HttpCacheEntry entry,
567             final Date now) {
568         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
569             for(final HeaderElement elt : h.getElements()) {
570                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
571                     try {
572                         final int maxstale = Integer.parseInt(elt.getValue());
573                         final long age = this.validityPolicy.getCurrentAgeSecs(entry, now);
574                         final long lifetime = this.validityPolicy.getFreshnessLifetimeSecs(entry);
575                         if (age - lifetime > maxstale) {
576                             return true;
577                         }
578                     } catch (final NumberFormatException nfe) {
579                         return true;
580                     }
581                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
582                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
583                     return true;
584                 }
585             }
586         }
587         return false;
588     }
589 
590     private String generateViaHeader(final HttpMessage msg) {
591 
592         final ProtocolVersion pv = msg.getProtocolVersion();
593         final String existingEntry = viaHeaders.get(pv);
594         if (existingEntry != null) {
595             return existingEntry;
596         }
597 
598         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
599         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
600 
601         final String value;
602         if ("http".equalsIgnoreCase(pv.getProtocol())) {
603             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getMajor(), pv.getMinor(),
604                     release);
605         } else {
606             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), pv.getMajor(),
607                     pv.getMinor(), release);
608         }
609         viaHeaders.put(pv, value);
610 
611         return value;
612     }
613 
614     private void setResponseStatus(final HttpCacheContext clientContext, final CacheResponseStatus value) {
615         if (clientContext != null) {
616             clientContext.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
617         }
618     }
619 
620     /**
621      * Reports whether this {@code CachingHttpClient} implementation
622      * supports byte-range requests as specified by the {@code Range}
623      * and {@code Content-Range} headers.
624      * @return {@code true} if byte-range requests are supported
625      */
626     public boolean supportsRangeAndContentRangeHeaders() {
627         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
628     }
629 
630     /**
631      * Reports whether this {@code CachingHttpClient} is configured as
632      * a shared (public) or non-shared (private) cache. See {@link
633      * CacheConfig#setSharedCache(boolean)}.
634      * @return {@code true} if we are behaving as a shared (public)
635      *   cache
636      */
637     public boolean isSharedCache() {
638         return this.sharedCache;
639     }
640 
641     Date getCurrentDate() {
642         return new Date();
643     }
644 
645     boolean clientRequestsOurOptions(final HttpRequest request) {
646         final RequestLine line = request.getRequestLine();
647 
648         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
649             return false;
650         }
651 
652         if (!"*".equals(line.getUri())) {
653             return false;
654         }
655         return "0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue());
656     }
657 
658     void callBackend(
659             final BasicFuture<HttpResponse> future,
660             final HttpHost target,
661             final HttpRequestWrapper request,
662             final HttpCacheContext clientContext) {
663         final Date requestDate = getCurrentDate();
664         this.log.trace("Calling the backend");
665 
666         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
667 
668             @Override
669             public void completed(final HttpResponse httpResponse) {
670                 httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
671                 try {
672                     final CloseableHttpResponse backendResponse = handleBackendResponse(
673                             target, request, requestDate, getCurrentDate(),
674                             Proxies.enhanceResponse(httpResponse));
675                     super.completed(backendResponse);
676                 } catch (final IOException e) {
677                     super.failed(e);
678                 }
679 
680             }
681 
682         };
683         this.backend.execute(target, request, clientContext, chainedFutureCallback);
684     }
685 
686     private boolean revalidationResponseIsTooOld(
687             final HttpResponse backendResponse,
688             final HttpCacheEntry cacheEntry) {
689         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
690         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
691         if (entryDateHeader != null && responseDateHeader != null) {
692             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
693             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
694             if (respDate != null && respDate.before(entryDate)) {
695                 return true;
696             }
697         }
698         return false;
699     }
700 
701     void negotiateResponseFromVariants(
702             final BasicFuture<HttpResponse> future,
703             final HttpHost target,
704             final HttpRequestWrapper request,
705             final HttpCacheContext clientContext,
706             final Map<String, Variant> variants) {
707         final HttpRequest conditionalRequest = this.conditionalRequestBuilder.buildConditionalRequestFromVariants(request, variants);
708 
709         final Date requestDate = getCurrentDate();
710 
711         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
712 
713             @Override
714             public void completed(final HttpResponse httpResponse) {
715                 final Date responseDate = getCurrentDate();
716 
717                 httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
718 
719                 if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
720                     try {
721                         future.completed(handleBackendResponse(
722                                 target, request, requestDate, responseDate,
723                                 Proxies.enhanceResponse(httpResponse)));
724                         return;
725                     } catch (final IOException e) {
726                         future.failed(e);
727                         return;
728                     }
729                 }
730 
731                 final Header resultEtagHeader = httpResponse.getFirstHeader(HeaderConstants.ETAG);
732                 if (resultEtagHeader == null) {
733                     CachingHttpAsyncClient.this.log.warn("304 response did not contain ETag");
734                     callBackend(future, target, request, clientContext);
735                     return;
736                 }
737 
738                 final String resultEtag = resultEtagHeader.getValue();
739                 final Variant matchingVariant = variants.get(resultEtag);
740                 if (matchingVariant == null) {
741                     CachingHttpAsyncClient.this.log.debug("304 response did not contain ETag matching one sent in If-None-Match");
742                     callBackend(future, target, request, clientContext);
743                 }
744 
745                 final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
746 
747                 if (revalidationResponseIsTooOld(httpResponse, matchedEntry)) {
748                     EntityUtils.consumeQuietly(httpResponse.getEntity());
749                     retryRequestUnconditionally(future, target, request, clientContext, matchedEntry);
750                     return;
751                 }
752 
753                 recordCacheUpdate(clientContext);
754 
755                 final HttpCacheEntry responseEntry = getUpdatedVariantEntry(target,
756                         conditionalRequest, requestDate, responseDate, httpResponse,
757                         matchingVariant, matchedEntry);
758 
759                 final HttpResponse resp = CachingHttpAsyncClient.this.responseGenerator.generateResponse(responseEntry);
760                 tryToUpdateVariantMap(target, request, matchingVariant);
761 
762                 if (shouldSendNotModifiedResponse(request, responseEntry)) {
763                     future.completed(CachingHttpAsyncClient.this.responseGenerator.generateNotModifiedResponse(responseEntry));
764                     return;
765                 }
766 
767                 future.completed(resp);
768             }
769 
770         };
771 
772         this.backend.execute(target, conditionalRequest, clientContext, chainedFutureCallback);
773     }
774 
775     private void retryRequestUnconditionally(
776             final BasicFuture<HttpResponse> future,
777             final HttpHost target,
778             final HttpRequestWrapper request,
779             final HttpCacheContext clientContext,
780             final HttpCacheEntry matchedEntry) {
781         final HttpRequestWrapper unconditional = this.conditionalRequestBuilder
782             .buildUnconditionalRequest(request, matchedEntry);
783         callBackend(future, target, unconditional, clientContext);
784     }
785 
786     private HttpCacheEntry getUpdatedVariantEntry(
787             final HttpHost target,
788             final HttpRequest conditionalRequest,
789             final Date requestDate,
790             final Date responseDate,
791             final HttpResponse backendResponse,
792             final Variant matchingVariant,
793             final HttpCacheEntry matchedEntry) {
794         HttpCacheEntry responseEntry = matchedEntry;
795         try {
796             responseEntry = this.responseCache.updateVariantCacheEntry(target, conditionalRequest,
797                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
798         } catch (final IOException ioe) {
799             this.log.warn("Could not update cache entry", ioe);
800         }
801         return responseEntry;
802     }
803 
804     private void tryToUpdateVariantMap(
805             final HttpHost target,
806             final HttpRequest request,
807             final Variant matchingVariant) {
808         try {
809             this.responseCache.reuseVariantEntryFor(target, request, matchingVariant);
810         } catch (final IOException ioe) {
811             this.log.warn("Could not update cache entry to reuse variant", ioe);
812         }
813     }
814 
815     private boolean shouldSendNotModifiedResponse(
816             final HttpRequest request,
817             final HttpCacheEntry responseEntry) {
818         return (this.suitabilityChecker.isConditional(request)
819                 && this.suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
820     }
821 
822     void revalidateCacheEntry(
823             final BasicFuture<HttpResponse> future,
824             final HttpHost target,
825             final HttpRequestWrapper request,
826             final HttpCacheContext clientContext,
827             final HttpCacheEntry cacheEntry) throws ProtocolException {
828 
829         final HttpRequestWrapper conditionalRequest = this.conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
830         final Date requestDate = getCurrentDate();
831 
832         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
833 
834             @Override
835             public void completed(final HttpResponse httpResponse) {
836                 final Date responseDate = getCurrentDate();
837 
838                 if (revalidationResponseIsTooOld(httpResponse, cacheEntry)) {
839                     final HttpRequest unconditional = CachingHttpAsyncClient.this.conditionalRequestBuilder.buildUnconditionalRequest(request, cacheEntry);
840                     final Date innerRequestDate = getCurrentDate();
841 
842                     final ChainedFutureCallback<HttpResponse> chainedFutureCallback2 = new ChainedFutureCallback<HttpResponse>(future) {
843 
844                         @Override
845                         public void completed(final HttpResponse innerHttpResponse) {
846                             final Date innerResponseDate = getCurrentDate();
847                             revalidateCacheEntryCompleted(future,
848                                     target, request, clientContext, cacheEntry,
849                                     conditionalRequest, innerRequestDate,
850                                     innerHttpResponse, innerResponseDate);
851                         }
852 
853                     };
854                     CachingHttpAsyncClient.this.backend.execute(
855                             target, unconditional, clientContext, chainedFutureCallback2);
856                 }
857 
858                 revalidateCacheEntryCompleted(future,
859                         target, request, clientContext, cacheEntry,
860                         conditionalRequest, requestDate,
861                         httpResponse, responseDate);
862             }
863 
864         };
865 
866         this.backend.execute(target, conditionalRequest, clientContext, chainedFutureCallback);
867     }
868 
869     private void revalidateCacheEntryCompleted(
870             final BasicFuture<HttpResponse> future,
871             final HttpHost target,
872             final HttpRequestWrapper request,
873             final HttpCacheContext clientContext,
874             final HttpCacheEntry cacheEntry,
875             final HttpRequestWrapper conditionalRequest,
876             final Date requestDate,
877             final HttpResponse httpResponse,
878             final Date responseDate) {
879 
880         httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
881 
882         final int statusCode = httpResponse.getStatusLine().getStatusCode();
883         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
884             recordCacheUpdate(clientContext);
885         }
886 
887         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
888             final HttpCacheEntry updatedEntry;
889             try {
890                 updatedEntry = CachingHttpAsyncClient.this.responseCache.updateCacheEntry(target, request, cacheEntry,
891                         httpResponse, requestDate, responseDate);
892             } catch (final IOException e) {
893                 future.failed(e);
894                 return;
895             }
896             if (CachingHttpAsyncClient.this.suitabilityChecker.isConditional(request)
897                     && CachingHttpAsyncClient.this.suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
898                 future.completed(CachingHttpAsyncClient.this.responseGenerator.generateNotModifiedResponse(updatedEntry));
899                 return;
900             }
901             future.completed(CachingHttpAsyncClient.this.responseGenerator.generateResponse(updatedEntry));
902             return;
903         }
904 
905         if (staleIfErrorAppliesTo(statusCode)
906             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
907             && CachingHttpAsyncClient.this.validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
908             final HttpResponse cachedResponse = CachingHttpAsyncClient.this.responseGenerator.generateResponse(cacheEntry);
909             cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
910             future.completed(cachedResponse);
911             return;
912         }
913 
914         try {
915             final CloseableHttpResponse backendResponse = handleBackendResponse(
916                     target, conditionalRequest, requestDate, responseDate,
917                     Proxies.enhanceResponse(httpResponse));
918             future.completed(backendResponse);
919         } catch (final IOException e) {
920             future.failed(e);
921         }
922     }
923 
924     private boolean staleIfErrorAppliesTo(final int statusCode) {
925         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
926                 || statusCode == HttpStatus.SC_BAD_GATEWAY
927                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
928                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
929     }
930 
931     CloseableHttpResponse handleBackendResponse(
932             final HttpHost target,
933             final HttpRequestWrapper request,
934             final Date requestDate,
935             final Date responseDate,
936             final CloseableHttpResponse backendResponse) throws IOException {
937 
938         this.log.debug("Handling Backend response");
939         this.responseCompliance.ensureProtocolCompliance(request, backendResponse);
940 
941         final boolean cacheable = this.responseCachingPolicy.isResponseCacheable(request, backendResponse);
942         this.responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
943         if (cacheable &&
944             !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
945             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
946             return this.responseCache.cacheAndReturnResponse(target, request, backendResponse, requestDate,
947                     responseDate);
948         }
949         if (!cacheable) {
950             try {
951                 this.responseCache.flushCacheEntriesFor(target, request);
952             } catch (final IOException ioe) {
953                 this.log.warn("Unable to flush invalid cache entries", ioe);
954             }
955         }
956         return backendResponse;
957     }
958 
959     /**
960      * For 304 Not modified responses, adds a "Last-Modified" header with the
961      * value of the "If-Modified-Since" header passed in the request. This
962      * header is required to be able to reuse match the cache entry for
963      * subsequent requests but as defined in http specifications it is not
964      * included in 304 responses by backend servers. This header will not be
965      * included in the resulting response.
966      */
967     private void storeRequestIfModifiedSinceFor304Response(
968             final HttpRequest request,
969             final HttpResponse backendResponse) {
970         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
971             final Header h = request.getFirstHeader("If-Modified-Since");
972             if (h != null) {
973                 backendResponse.addHeader("Last-Modified", h.getValue());
974             }
975         }
976     }
977 
978     private boolean alreadyHaveNewerCacheEntry(
979             final HttpHost target,
980             final HttpRequest request,
981             final HttpResponse backendResponse) {
982         HttpCacheEntry existing = null;
983         try {
984             existing = this.responseCache.getCacheEntry(target, request);
985         } catch (final IOException ioe) {
986             // nop
987         }
988         if (existing == null) {
989             return false;
990         }
991         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
992         if (entryDateHeader == null) {
993             return false;
994         }
995         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
996         if (responseDateHeader == null) {
997             return false;
998         }
999         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
1000         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
1001         return responseDate != null && responseDate.before(entryDate);
1002     }
1003 
1004 }