View Javadoc

1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.net.URI;
31  import java.util.Date;
32  import java.util.HashMap;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.concurrent.Future;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.http.Header;
41  import org.apache.http.HeaderElement;
42  import org.apache.http.HttpHost;
43  import org.apache.http.HttpMessage;
44  import org.apache.http.HttpRequest;
45  import org.apache.http.HttpResponse;
46  import org.apache.http.HttpStatus;
47  import org.apache.http.HttpVersion;
48  import org.apache.http.ProtocolException;
49  import org.apache.http.ProtocolVersion;
50  import org.apache.http.RequestLine;
51  import org.apache.http.annotation.ThreadSafe;
52  import org.apache.http.client.ClientProtocolException;
53  import org.apache.http.client.cache.CacheResponseStatus;
54  import org.apache.http.client.cache.HeaderConstants;
55  import org.apache.http.client.cache.HttpCacheContext;
56  import org.apache.http.client.cache.HttpCacheEntry;
57  import org.apache.http.client.cache.HttpCacheStorage;
58  import org.apache.http.client.cache.ResourceFactory;
59  import org.apache.http.client.methods.CloseableHttpResponse;
60  import org.apache.http.client.methods.HttpRequestWrapper;
61  import org.apache.http.client.methods.HttpUriRequest;
62  import org.apache.http.client.protocol.HttpClientContext;
63  import org.apache.http.client.utils.DateUtils;
64  import org.apache.http.concurrent.BasicFuture;
65  import org.apache.http.concurrent.FutureCallback;
66  import org.apache.http.conn.routing.HttpRoute;
67  import org.apache.http.impl.nio.client.HttpAsyncClients;
68  import org.apache.http.message.BasicHttpResponse;
69  import org.apache.http.nio.client.HttpAsyncClient;
70  import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
71  import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
72  import org.apache.http.nio.reactor.IOReactorException;
73  import org.apache.http.protocol.HTTP;
74  import org.apache.http.protocol.HttpContext;
75  import org.apache.http.util.Args;
76  import org.apache.http.util.EntityUtils;
77  import org.apache.http.util.VersionInfo;
78  
79  @ThreadSafe // So long as the responseCache implementation is threadsafe
80  public class CachingHttpAsyncClient implements HttpAsyncClient {
81  
82      private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
83  
84      private final AtomicLong cacheHits = new AtomicLong();
85      private final AtomicLong cacheMisses = new AtomicLong();
86      private final AtomicLong cacheUpdates = new AtomicLong();
87  
88      private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
89  
90      private final HttpAsyncClient backend;
91      private final HttpCache responseCache;
92      private final CacheValidityPolicy validityPolicy;
93      private final ResponseCachingPolicy responseCachingPolicy;
94      private final CachedHttpResponseGenerator responseGenerator;
95      private final CacheableRequestPolicy cacheableRequestPolicy;
96      private final CachedResponseSuitabilityChecker suitabilityChecker;
97  
98      private final ConditionalRequestBuilder conditionalRequestBuilder;
99  
100     private final long maxObjectSizeBytes;
101     private final boolean sharedCache;
102 
103     private final ResponseProtocolCompliance responseCompliance;
104     private final RequestProtocolCompliance requestCompliance;
105 
106     private final AsynchronousAsyncValidator asynchAsyncRevalidator;
107 
108     private final Log log = LogFactory.getLog(getClass());
109 
110     CachingHttpAsyncClient(
111             final HttpAsyncClient client,
112             final HttpCache cache,
113             final CacheConfig config) {
114         super();
115         Args.notNull(client, "HttpClient");
116         Args.notNull(cache, "HttpCache");
117         Args.notNull(config, "CacheConfig");
118         this.maxObjectSizeBytes = config.getMaxObjectSize();
119         this.sharedCache = config.isSharedCache();
120         this.backend = client;
121         this.responseCache = cache;
122         this.validityPolicy = new CacheValidityPolicy();
123         this.responseCachingPolicy = new ResponseCachingPolicy(this.maxObjectSizeBytes, this.sharedCache, false, config.is303CachingEnabled());
124         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
125         this.cacheableRequestPolicy = new CacheableRequestPolicy();
126         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, config);
127         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
128 
129         this.responseCompliance = new ResponseProtocolCompliance();
130         this.requestCompliance = new RequestProtocolCompliance(config.isWeakETagOnPutDeleteAllowed());
131 
132         this.asynchAsyncRevalidator = makeAsynchronousValidator(config);
133     }
134 
135     public CachingHttpAsyncClient() throws IOReactorException {
136         this(HttpAsyncClients.createDefault(),
137                 new BasicHttpCache(),
138                 CacheConfig.DEFAULT);
139     }
140 
141     public CachingHttpAsyncClient(final CacheConfig config) throws IOReactorException {
142         this(HttpAsyncClients.createDefault(),
143                 new BasicHttpCache(config),
144                 config);
145     }
146 
147     public CachingHttpAsyncClient(final HttpAsyncClient client) {
148         this(client,
149                 new BasicHttpCache(),
150                 CacheConfig.DEFAULT);
151     }
152 
153     public CachingHttpAsyncClient(final HttpAsyncClient client, final CacheConfig config) {
154         this(client,
155                 new BasicHttpCache(config),
156                 config);
157     }
158 
159     public CachingHttpAsyncClient(
160             final HttpAsyncClient client,
161             final ResourceFactory resourceFactory,
162             final HttpCacheStorage storage,
163             final CacheConfig config) {
164         this(client,
165                 new BasicHttpCache(resourceFactory, storage, config),
166                 config);
167     }
168 
169     public CachingHttpAsyncClient(
170             final HttpAsyncClient client,
171             final HttpCacheStorage storage,
172             final CacheConfig config) {
173         this(client,
174                 new BasicHttpCache(new HeapResourceFactory(), storage, config),
175                 config);
176     }
177 
178     CachingHttpAsyncClient(
179             final HttpAsyncClient backend,
180             final CacheValidityPolicy validityPolicy,
181             final ResponseCachingPolicy responseCachingPolicy,
182             final HttpCache responseCache,
183             final CachedHttpResponseGenerator responseGenerator,
184             final CacheableRequestPolicy cacheableRequestPolicy,
185             final CachedResponseSuitabilityChecker suitabilityChecker,
186             final ConditionalRequestBuilder conditionalRequestBuilder,
187             final ResponseProtocolCompliance responseCompliance,
188             final RequestProtocolCompliance requestCompliance) {
189         final CacheConfig config = CacheConfig.DEFAULT;
190         this.maxObjectSizeBytes = config.getMaxObjectSize();
191         this.sharedCache = config.isSharedCache();
192         this.backend = backend;
193         this.validityPolicy = validityPolicy;
194         this.responseCachingPolicy = responseCachingPolicy;
195         this.responseCache = responseCache;
196         this.responseGenerator = responseGenerator;
197         this.cacheableRequestPolicy = cacheableRequestPolicy;
198         this.suitabilityChecker = suitabilityChecker;
199         this.conditionalRequestBuilder = conditionalRequestBuilder;
200         this.responseCompliance = responseCompliance;
201         this.requestCompliance = requestCompliance;
202         this.asynchAsyncRevalidator = makeAsynchronousValidator(config);
203     }
204 
205     private AsynchronousAsyncValidator makeAsynchronousValidator(
206             final CacheConfig config) {
207         if (config.getAsynchronousWorkersMax() > 0) {
208             return new AsynchronousAsyncValidator(this, config);
209         }
210         return null;
211     }
212 
213     /**
214      * Reports the number of times that the cache successfully responded
215      * to an {@link HttpRequest} without contacting the origin server.
216      * @return the number of cache hits
217      */
218     public long getCacheHits() {
219         return this.cacheHits.get();
220     }
221 
222     /**
223      * Reports the number of times that the cache contacted the origin
224      * server because it had no appropriate response cached.
225      * @return the number of cache misses
226      */
227     public long getCacheMisses() {
228         return this.cacheMisses.get();
229     }
230 
231     /**
232      * Reports the number of times that the cache was able to satisfy
233      * a response by revalidating an existing but stale cache entry.
234      * @return the number of cache revalidations
235      */
236     public long getCacheUpdates() {
237         return this.cacheUpdates.get();
238     }
239 
240     @Override
241     public Future<HttpResponse> execute(
242             final HttpHost target,
243             final HttpRequest request,
244             final FutureCallback<HttpResponse> callback) {
245         return execute(target, request, null, callback);
246     }
247 
248     @Override
249     public <T> Future<T> execute(
250             final HttpAsyncRequestProducer requestProducer,
251             final HttpAsyncResponseConsumer<T> responseConsumer,
252             final FutureCallback<T> callback) {
253         return execute(requestProducer, responseConsumer, null, callback);
254     }
255 
256     @Override
257     public <T> Future<T> execute(
258             final HttpAsyncRequestProducer requestProducer,
259             final HttpAsyncResponseConsumer<T> responseConsumer,
260             final HttpContext context,
261             final FutureCallback<T> callback) {
262         this.log.warn("CachingHttpAsyncClient does not support caching for streaming HTTP exchanges");
263         return this.backend.execute(requestProducer, responseConsumer, context, callback);
264     }
265 
266     @Override
267     public Future<HttpResponse> execute(
268             final HttpUriRequest request,
269             final FutureCallback<HttpResponse> callback) {
270         return execute(request, null, callback);
271     }
272 
273     @Override
274     public Future<HttpResponse> execute(
275             final HttpUriRequest request,
276             final HttpContext context,
277             final FutureCallback<HttpResponse> callback) {
278         final URI uri = request.getURI();
279         final HttpHost httpHost = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
280         return execute(httpHost, request, context, callback);
281     }
282 
283     @Override
284     public Future<HttpResponse> execute(
285             final HttpHost target,
286             final HttpRequest originalRequest,
287             final HttpContext context,
288             final FutureCallback<HttpResponse> futureCallback) {
289         final BasicFuture<HttpResponse> future = new BasicFuture<HttpResponse>(futureCallback);
290         final HttpRequestWrapper request = HttpRequestWrapper.wrap(originalRequest);
291         final HttpCacheContext clientContext = HttpCacheContext.adapt(context);
292         // default response context
293         setResponseStatus(clientContext, CacheResponseStatus.CACHE_MISS);
294 
295         final String via = generateViaHeader(request);
296 
297         if (clientRequestsOurOptions(request)) {
298             setResponseStatus(clientContext, CacheResponseStatus.CACHE_MODULE_RESPONSE);
299             future.completed(new OptionsHttp11Response());
300             return future;
301         }
302 
303         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(
304                 request, clientContext);
305         if (fatalErrorResponse != null) {
306             future.completed(fatalErrorResponse);
307             return future;
308         }
309 
310         try {
311             this.requestCompliance.makeRequestCompliant(request);
312         } catch (final ClientProtocolException e) {
313             future.failed(e);
314             return future;
315         }
316         request.addHeader(HeaderConstants.VIA,via);
317 
318         flushEntriesInvalidatedByRequest(target, request);
319 
320         if (!this.cacheableRequestPolicy.isServableFromCache(request)) {
321             log.debug("Request is not servable from cache");
322             callBackend(future, target, request, clientContext);
323             return future;
324         }
325 
326         final HttpCacheEntry entry = satisfyFromCache(target, request);
327         if (entry == null) {
328             log.debug("Cache miss");
329             handleCacheMiss(future, target, request, clientContext);
330         } else {
331             try {
332                 handleCacheHit(future, target, request, clientContext, entry);
333             } catch (final IOException e) {
334                 future.failed(e);
335             }
336         }
337         return future;
338     }
339 
340     private void handleCacheHit(
341             final BasicFuture<HttpResponse> future,
342             final HttpHost target,
343             final HttpRequestWrapper request,
344             final HttpCacheContext clientContext,
345             final HttpCacheEntry entry) throws IOException {
346         recordCacheHit(target, request);
347         final HttpResponse out;
348         final Date now = getCurrentDate();
349         if (this.suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
350             log.debug("Cache hit");
351             out = generateCachedResponse(request, clientContext, entry, now);
352         } else if (!mayCallBackend(request)) {
353             log.debug("Cache entry not suitable but only-if-cached requested");
354             out = generateGatewayTimeout(clientContext);
355         } else if (validityPolicy.isRevalidatable(entry)
356                 && !(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
357                 && !suitabilityChecker.isConditional(request))) {
358             log.debug("Revalidating cache entry");
359             revalidateCacheEntry(future, target, request, clientContext, entry, now);
360             return;
361         } else {
362             log.debug("Cache entry not usable; calling backend");
363             callBackend(future, target, request, clientContext);
364             return;
365         }
366         clientContext.setAttribute(HttpClientContext.HTTP_ROUTE, new HttpRoute(target));
367         clientContext.setAttribute(HttpClientContext.HTTP_TARGET_HOST, target);
368         clientContext.setAttribute(HttpClientContext.HTTP_REQUEST, request);
369         clientContext.setAttribute(HttpClientContext.HTTP_RESPONSE, out);
370         clientContext.setAttribute(HttpClientContext.HTTP_REQ_SENT, Boolean.TRUE);
371         future.completed(out);
372     }
373 
374     private void revalidateCacheEntry(
375             final BasicFuture<HttpResponse> future,
376             final HttpHost target,
377             final HttpRequestWrapper request,
378             final HttpCacheContext clientContext,
379             final HttpCacheEntry entry,
380             final Date now) throws ClientProtocolException {
381 
382         try {
383             if (this.asynchAsyncRevalidator != null
384                 && !staleResponseNotAllowed(request, entry, now)
385                 && this.validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
386                 this.log.debug("Serving stale with asynchronous revalidation");
387                 final HttpResponse resp = this.responseGenerator.generateResponse(request, entry);
388                 resp.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
389 
390                 this.asynchAsyncRevalidator.revalidateCacheEntry(target, request, clientContext, entry);
391 
392                 future.completed(resp);
393                 return;
394             }
395 
396             final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
397 
398                 @Override
399                 public void failed(final Exception ex) {
400                     if(ex instanceof IOException) {
401                         super.completed(handleRevalidationFailure(request, clientContext, entry, now));
402                     } else {
403                         super.failed(ex);
404                     }
405                 }
406 
407             };
408 
409             final BasicFuture<HttpResponse> compositeFuture = new BasicFuture<HttpResponse>(chainedFutureCallback);
410             revalidateCacheEntry(compositeFuture, target, request, clientContext, entry);
411         } catch (final ProtocolException e) {
412             throw new ClientProtocolException(e);
413         }
414     }
415 
416     private void handleCacheMiss(
417             final BasicFuture<HttpResponse> future,
418             final HttpHost target,
419             final HttpRequestWrapper request,
420             final HttpCacheContext clientContext) {
421         recordCacheMiss(target, request);
422 
423         if (!mayCallBackend(request)) {
424             future.completed(new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
425             return;
426         }
427 
428         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
429         if (variants != null && variants.size() > 0) {
430             negotiateResponseFromVariants(future, target, request, clientContext, variants);
431             return;
432         }
433 
434         callBackend(future, target, request, clientContext);
435     }
436 
437     private HttpCacheEntry satisfyFromCache(
438             final HttpHost target,
439             final HttpRequest request) {
440         HttpCacheEntry entry = null;
441         try {
442             entry = this.responseCache.getCacheEntry(target, request);
443         } catch (final IOException ioe) {
444             this.log.warn("Unable to retrieve entries from cache", ioe);
445         }
446         return entry;
447     }
448 
449     private HttpResponse getFatallyNoncompliantResponse(
450             final HttpRequest request,
451             final HttpCacheContext clientContext) {
452         HttpResponse fatalErrorResponse = null;
453         final List<RequestProtocolError> fatalError = this.requestCompliance.requestIsFatallyNonCompliant(request);
454 
455         for (final RequestProtocolError error : fatalError) {
456             setResponseStatus(clientContext, CacheResponseStatus.CACHE_MODULE_RESPONSE);
457             fatalErrorResponse = this.requestCompliance.getErrorForRequest(error);
458         }
459         return fatalErrorResponse;
460     }
461 
462     private Map<String, Variant> getExistingCacheVariants(
463             final HttpHost target,
464             final HttpRequest request) {
465         Map<String,Variant> variants = null;
466         try {
467             variants = this.responseCache.getVariantCacheEntriesWithEtags(target, request);
468         } catch (final IOException ioe) {
469             this.log.warn("Unable to retrieve variant entries from cache", ioe);
470         }
471         return variants;
472     }
473 
474     private void recordCacheMiss(final HttpHost target, final HttpRequest request) {
475         this.cacheMisses.getAndIncrement();
476         if (this.log.isDebugEnabled()) {
477             final RequestLine rl = request.getRequestLine();
478             this.log.debug("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
479         }
480     }
481 
482     private void recordCacheHit(final HttpHost target, final HttpRequest request) {
483         this.cacheHits.getAndIncrement();
484         if (this.log.isDebugEnabled()) {
485             final RequestLine rl = request.getRequestLine();
486             this.log.debug("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
487         }
488     }
489 
490     private void recordCacheUpdate(final HttpCacheContext clientContext) {
491         this.cacheUpdates.getAndIncrement();
492         setResponseStatus(clientContext, CacheResponseStatus.VALIDATED);
493     }
494 
495     private void flushEntriesInvalidatedByRequest(final HttpHost target,
496             final HttpRequest request) {
497         try {
498             this.responseCache.flushInvalidatedCacheEntriesFor(target, request);
499         } catch (final IOException ioe) {
500             this.log.warn("Unable to flush invalidated entries from cache", ioe);
501         }
502     }
503 
504     private HttpResponse generateCachedResponse(
505             final HttpRequestWrapper request,
506             final HttpCacheContext clientContext,
507             final HttpCacheEntry entry,
508             final Date now) {
509         final HttpResponse cachedResponse;
510         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
511                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
512             cachedResponse = this.responseGenerator.generateNotModifiedResponse(entry);
513         } else {
514             cachedResponse = this.responseGenerator.generateResponse(request, entry);
515         }
516         setResponseStatus(clientContext, CacheResponseStatus.CACHE_HIT);
517         if (this.validityPolicy.getStalenessSecs(entry, now) > 0L) {
518             cachedResponse.addHeader("Warning","110 localhost \"Response is stale\"");
519         }
520         return cachedResponse;
521     }
522 
523     private HttpResponse handleRevalidationFailure(
524             final HttpRequestWrapper request,
525             final HttpCacheContext clientContext,
526             final HttpCacheEntry entry,
527             final Date now) {
528         if (staleResponseNotAllowed(request, entry, now)) {
529             return generateGatewayTimeout(clientContext);
530         }
531         return unvalidatedCacheHit(clientContext, request, entry);
532     }
533 
534     private HttpResponse generateGatewayTimeout(final HttpCacheContext clientContext) {
535         setResponseStatus(clientContext, CacheResponseStatus.CACHE_MODULE_RESPONSE);
536         return new BasicHttpResponse(HttpVersion.HTTP_1_1,
537                 HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout");
538     }
539 
540     private HttpResponse unvalidatedCacheHit(
541             final HttpCacheContext clientContext,
542             final HttpRequestWrapper request,
543             final HttpCacheEntry entry) {
544         final HttpResponse cachedResponse = this.responseGenerator.generateResponse(request, entry);
545         setResponseStatus(clientContext, CacheResponseStatus.CACHE_HIT);
546         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
547         return cachedResponse;
548     }
549 
550     private boolean staleResponseNotAllowed(
551             final HttpRequest request,
552             final HttpCacheEntry entry,
553             final Date now) {
554         return this.validityPolicy.mustRevalidate(entry)
555             || (isSharedCache() && this.validityPolicy.proxyRevalidate(entry))
556             || explicitFreshnessRequest(request, entry, now);
557     }
558 
559     private boolean mayCallBackend(final HttpRequest request) {
560         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
561             for (final HeaderElement elt : h.getElements()) {
562                 if ("only-if-cached".equals(elt.getName())) {
563                     this.log.debug("Request marked only-if-cached");
564                     return false;
565                 }
566             }
567         }
568         return true;
569     }
570 
571     private boolean explicitFreshnessRequest(
572             final HttpRequest request,
573             final HttpCacheEntry entry,
574             final Date now) {
575         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
576             for(final HeaderElement elt : h.getElements()) {
577                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
578                     try {
579                         final int maxstale = Integer.parseInt(elt.getValue());
580                         final long age = this.validityPolicy.getCurrentAgeSecs(entry, now);
581                         final long lifetime = this.validityPolicy.getFreshnessLifetimeSecs(entry);
582                         if (age - lifetime > maxstale) {
583                             return true;
584                         }
585                     } catch (final NumberFormatException nfe) {
586                         return true;
587                     }
588                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
589                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
590                     return true;
591                 }
592             }
593         }
594         return false;
595     }
596 
597     private String generateViaHeader(final HttpMessage msg) {
598 
599         final ProtocolVersion pv = msg.getProtocolVersion();
600         final String existingEntry = viaHeaders.get(pv);
601         if (existingEntry != null) {
602             return existingEntry;
603         }
604 
605         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
606         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
607 
608         final String value;
609         if ("http".equalsIgnoreCase(pv.getProtocol())) {
610             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getMajor(), pv.getMinor(),
611                     release);
612         } else {
613             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), pv.getMajor(),
614                     pv.getMinor(), release);
615         }
616         viaHeaders.put(pv, value);
617 
618         return value;
619     }
620 
621     private void setResponseStatus(final HttpCacheContext clientContext, final CacheResponseStatus value) {
622         if (clientContext != null) {
623             clientContext.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
624         }
625     }
626 
627     /**
628      * Reports whether this {@code CachingHttpClient} implementation
629      * supports byte-range requests as specified by the {@code Range}
630      * and {@code Content-Range} headers.
631      * @return {@code true} if byte-range requests are supported
632      */
633     public boolean supportsRangeAndContentRangeHeaders() {
634         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
635     }
636 
637     /**
638      * Reports whether this {@code CachingHttpClient} is configured as
639      * a shared (public) or non-shared (private) cache. See {@link
640      * CacheConfig#setSharedCache(boolean)}.
641      * @return {@code true} if we are behaving as a shared (public)
642      *   cache
643      */
644     public boolean isSharedCache() {
645         return this.sharedCache;
646     }
647 
648     Date getCurrentDate() {
649         return new Date();
650     }
651 
652     boolean clientRequestsOurOptions(final HttpRequest request) {
653         final RequestLine line = request.getRequestLine();
654 
655         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
656             return false;
657         }
658 
659         if (!"*".equals(line.getUri())) {
660             return false;
661         }
662         return "0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue());
663     }
664 
665     void callBackend(
666             final BasicFuture<HttpResponse> future,
667             final HttpHost target,
668             final HttpRequestWrapper request,
669             final HttpCacheContext clientContext) {
670         final Date requestDate = getCurrentDate();
671         this.log.trace("Calling the backend");
672 
673         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
674 
675             @Override
676             public void completed(final HttpResponse httpResponse) {
677                 httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
678                 try {
679                     final CloseableHttpResponse backendResponse = handleBackendResponse(
680                             target, request, requestDate, getCurrentDate(),
681                             Proxies.enhanceResponse(httpResponse));
682                     super.completed(backendResponse);
683                 } catch (final IOException e) {
684                     super.failed(e);
685                 }
686 
687             }
688 
689         };
690         this.backend.execute(target, request, clientContext, chainedFutureCallback);
691     }
692 
693     private boolean revalidationResponseIsTooOld(
694             final HttpResponse backendResponse,
695             final HttpCacheEntry cacheEntry) {
696         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
697         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
698         if (entryDateHeader != null && responseDateHeader != null) {
699             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
700             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
701             if (respDate != null && respDate.before(entryDate)) {
702                 return true;
703             }
704         }
705         return false;
706     }
707 
708     void negotiateResponseFromVariants(
709             final BasicFuture<HttpResponse> future,
710             final HttpHost target,
711             final HttpRequestWrapper request,
712             final HttpCacheContext clientContext,
713             final Map<String, Variant> variants) {
714         final HttpRequest conditionalRequest = this.conditionalRequestBuilder.buildConditionalRequestFromVariants(request, variants);
715 
716         final Date requestDate = getCurrentDate();
717 
718         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
719 
720             @Override
721             public void completed(final HttpResponse httpResponse) {
722                 final Date responseDate = getCurrentDate();
723 
724                 httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
725 
726                 if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
727                     try {
728                         future.completed(handleBackendResponse(
729                                 target, request, requestDate, responseDate,
730                                 Proxies.enhanceResponse(httpResponse)));
731                         return;
732                     } catch (final IOException e) {
733                         future.failed(e);
734                         return;
735                     }
736                 }
737 
738                 final Header resultEtagHeader = httpResponse.getFirstHeader(HeaderConstants.ETAG);
739                 if (resultEtagHeader == null) {
740                     CachingHttpAsyncClient.this.log.warn("304 response did not contain ETag");
741                     callBackend(future, target, request, clientContext);
742                     return;
743                 }
744 
745                 final String resultEtag = resultEtagHeader.getValue();
746                 final Variant matchingVariant = variants.get(resultEtag);
747                 if (matchingVariant == null) {
748                     CachingHttpAsyncClient.this.log.debug("304 response did not contain ETag matching one sent in If-None-Match");
749                     callBackend(future, target, request, clientContext);
750                 }
751 
752                 final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
753 
754                 if (revalidationResponseIsTooOld(httpResponse, matchedEntry)) {
755                     EntityUtils.consumeQuietly(httpResponse.getEntity());
756                     retryRequestUnconditionally(future, target, request, clientContext, matchedEntry);
757                     return;
758                 }
759 
760                 recordCacheUpdate(clientContext);
761 
762                 final HttpCacheEntry responseEntry = getUpdatedVariantEntry(target,
763                         conditionalRequest, requestDate, responseDate, httpResponse,
764                         matchingVariant, matchedEntry);
765 
766                 final HttpResponse resp = CachingHttpAsyncClient.this.responseGenerator.generateResponse(request, responseEntry);
767                 tryToUpdateVariantMap(target, request, matchingVariant);
768 
769                 if (shouldSendNotModifiedResponse(request, responseEntry)) {
770                     future.completed(CachingHttpAsyncClient.this.responseGenerator.generateNotModifiedResponse(responseEntry));
771                     return;
772                 }
773 
774                 future.completed(resp);
775             }
776 
777         };
778 
779         this.backend.execute(target, conditionalRequest, clientContext, chainedFutureCallback);
780     }
781 
782     private void retryRequestUnconditionally(
783             final BasicFuture<HttpResponse> future,
784             final HttpHost target,
785             final HttpRequestWrapper request,
786             final HttpCacheContext clientContext,
787             final HttpCacheEntry matchedEntry) {
788         final HttpRequestWrapper unconditional = this.conditionalRequestBuilder
789             .buildUnconditionalRequest(request, matchedEntry);
790         callBackend(future, target, unconditional, clientContext);
791     }
792 
793     private HttpCacheEntry getUpdatedVariantEntry(
794             final HttpHost target,
795             final HttpRequest conditionalRequest,
796             final Date requestDate,
797             final Date responseDate,
798             final HttpResponse backendResponse,
799             final Variant matchingVariant,
800             final HttpCacheEntry matchedEntry) {
801         HttpCacheEntry responseEntry = matchedEntry;
802         try {
803             responseEntry = this.responseCache.updateVariantCacheEntry(target, conditionalRequest,
804                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
805         } catch (final IOException ioe) {
806             this.log.warn("Could not update cache entry", ioe);
807         }
808         return responseEntry;
809     }
810 
811     private void tryToUpdateVariantMap(
812             final HttpHost target,
813             final HttpRequest request,
814             final Variant matchingVariant) {
815         try {
816             this.responseCache.reuseVariantEntryFor(target, request, matchingVariant);
817         } catch (final IOException ioe) {
818             this.log.warn("Could not update cache entry to reuse variant", ioe);
819         }
820     }
821 
822     private boolean shouldSendNotModifiedResponse(
823             final HttpRequest request,
824             final HttpCacheEntry responseEntry) {
825         return (this.suitabilityChecker.isConditional(request)
826                 && this.suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
827     }
828 
829     void revalidateCacheEntry(
830             final BasicFuture<HttpResponse> future,
831             final HttpHost target,
832             final HttpRequestWrapper request,
833             final HttpCacheContext clientContext,
834             final HttpCacheEntry cacheEntry) throws ProtocolException {
835 
836         final HttpRequestWrapper conditionalRequest = this.conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
837         final Date requestDate = getCurrentDate();
838 
839         final ChainedFutureCallback<HttpResponse> chainedFutureCallback = new ChainedFutureCallback<HttpResponse>(future) {
840 
841             @Override
842             public void completed(final HttpResponse httpResponse) {
843                 final Date responseDate = getCurrentDate();
844 
845                 if (revalidationResponseIsTooOld(httpResponse, cacheEntry)) {
846                     final HttpRequest unconditional = CachingHttpAsyncClient.this.conditionalRequestBuilder.buildUnconditionalRequest(request, cacheEntry);
847                     final Date innerRequestDate = getCurrentDate();
848 
849                     final ChainedFutureCallback<HttpResponse> chainedFutureCallback2 = new ChainedFutureCallback<HttpResponse>(future) {
850 
851                         @Override
852                         public void completed(final HttpResponse innerHttpResponse) {
853                             final Date innerResponseDate = getCurrentDate();
854                             revalidateCacheEntryCompleted(future,
855                                     target, request, clientContext, cacheEntry,
856                                     conditionalRequest, innerRequestDate,
857                                     innerHttpResponse, innerResponseDate);
858                         }
859 
860                     };
861                     CachingHttpAsyncClient.this.backend.execute(
862                             target, unconditional, clientContext, chainedFutureCallback2);
863                 }
864 
865                 revalidateCacheEntryCompleted(future,
866                         target, request, clientContext, cacheEntry,
867                         conditionalRequest, requestDate,
868                         httpResponse, responseDate);
869             }
870 
871         };
872 
873         this.backend.execute(target, conditionalRequest, clientContext, chainedFutureCallback);
874     }
875 
876     private void revalidateCacheEntryCompleted(
877             final BasicFuture<HttpResponse> future,
878             final HttpHost target,
879             final HttpRequestWrapper request,
880             final HttpCacheContext clientContext,
881             final HttpCacheEntry cacheEntry,
882             final HttpRequestWrapper conditionalRequest,
883             final Date requestDate,
884             final HttpResponse httpResponse,
885             final Date responseDate) {
886 
887         httpResponse.addHeader(HeaderConstants.VIA, generateViaHeader(httpResponse));
888 
889         final int statusCode = httpResponse.getStatusLine().getStatusCode();
890         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
891             recordCacheUpdate(clientContext);
892         }
893 
894         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
895             final HttpCacheEntry updatedEntry;
896             try {
897                 updatedEntry = CachingHttpAsyncClient.this.responseCache.updateCacheEntry(target, request, cacheEntry,
898                         httpResponse, requestDate, responseDate);
899             } catch (final IOException e) {
900                 future.failed(e);
901                 return;
902             }
903             if (CachingHttpAsyncClient.this.suitabilityChecker.isConditional(request)
904                     && CachingHttpAsyncClient.this.suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
905                 future.completed(CachingHttpAsyncClient.this.responseGenerator.generateNotModifiedResponse(updatedEntry));
906                 return;
907             }
908             future.completed(CachingHttpAsyncClient.this.responseGenerator.generateResponse(request, updatedEntry));
909             return;
910         }
911 
912         if (staleIfErrorAppliesTo(statusCode)
913             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
914             && CachingHttpAsyncClient.this.validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
915             final HttpResponse cachedResponse = CachingHttpAsyncClient.this.responseGenerator.generateResponse(request, cacheEntry);
916             cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
917             future.completed(cachedResponse);
918             return;
919         }
920 
921         try {
922             final CloseableHttpResponse backendResponse = handleBackendResponse(
923                     target, conditionalRequest, requestDate, responseDate,
924                     Proxies.enhanceResponse(httpResponse));
925             future.completed(backendResponse);
926         } catch (final IOException e) {
927             future.failed(e);
928         }
929     }
930 
931     private boolean staleIfErrorAppliesTo(final int statusCode) {
932         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
933                 || statusCode == HttpStatus.SC_BAD_GATEWAY
934                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
935                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
936     }
937 
938     CloseableHttpResponse handleBackendResponse(
939             final HttpHost target,
940             final HttpRequestWrapper request,
941             final Date requestDate,
942             final Date responseDate,
943             final CloseableHttpResponse backendResponse) throws IOException {
944 
945         this.log.debug("Handling Backend response");
946         this.responseCompliance.ensureProtocolCompliance(request, backendResponse);
947 
948         final boolean cacheable = this.responseCachingPolicy.isResponseCacheable(request, backendResponse);
949         this.responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
950         if (cacheable &&
951             !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
952             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
953             return this.responseCache.cacheAndReturnResponse(target, request, backendResponse, requestDate,
954                     responseDate);
955         }
956         if (!cacheable) {
957             try {
958                 this.responseCache.flushCacheEntriesFor(target, request);
959             } catch (final IOException ioe) {
960                 this.log.warn("Unable to flush invalid cache entries", ioe);
961             }
962         }
963         return backendResponse;
964     }
965 
966     /**
967      * For 304 Not modified responses, adds a "Last-Modified" header with the
968      * value of the "If-Modified-Since" header passed in the request. This
969      * header is required to be able to reuse match the cache entry for
970      * subsequent requests but as defined in http specifications it is not
971      * included in 304 responses by backend servers. This header will not be
972      * included in the resulting response.
973      */
974     private void storeRequestIfModifiedSinceFor304Response(
975             final HttpRequest request,
976             final HttpResponse backendResponse) {
977         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
978             final Header h = request.getFirstHeader("If-Modified-Since");
979             if (h != null) {
980                 backendResponse.addHeader("Last-Modified", h.getValue());
981             }
982         }
983     }
984 
985     private boolean alreadyHaveNewerCacheEntry(
986             final HttpHost target,
987             final HttpRequest request,
988             final HttpResponse backendResponse) {
989         HttpCacheEntry existing = null;
990         try {
991             existing = this.responseCache.getCacheEntry(target, request);
992         } catch (final IOException ioe) {
993             // nop
994         }
995         if (existing == null) {
996             return false;
997         }
998         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
999         if (entryDateHeader == null) {
1000             return false;
1001         }
1002         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
1003         if (responseDateHeader == null) {
1004             return false;
1005         }
1006         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
1007         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
1008         return responseDate != null && responseDate.before(entryDate);
1009     }
1010 
1011 }