View Javadoc
1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.net.URI;
31  import java.net.URISyntaxException;
32  import java.util.Date;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.http.Header;
41  import org.apache.http.HeaderElement;
42  import org.apache.http.HttpException;
43  import org.apache.http.HttpHost;
44  import org.apache.http.HttpMessage;
45  import org.apache.http.HttpRequest;
46  import org.apache.http.HttpResponse;
47  import org.apache.http.HttpStatus;
48  import org.apache.http.HttpVersion;
49  import org.apache.http.ProtocolException;
50  import org.apache.http.ProtocolVersion;
51  import org.apache.http.RequestLine;
52  import org.apache.http.annotation.Contract;
53  import org.apache.http.annotation.ThreadingBehavior;
54  import org.apache.http.client.cache.CacheResponseStatus;
55  import org.apache.http.client.cache.HeaderConstants;
56  import org.apache.http.client.cache.HttpCacheContext;
57  import org.apache.http.client.cache.HttpCacheEntry;
58  import org.apache.http.client.cache.HttpCacheStorage;
59  import org.apache.http.client.cache.ResourceFactory;
60  import org.apache.http.client.methods.CloseableHttpResponse;
61  import org.apache.http.client.methods.HttpExecutionAware;
62  import org.apache.http.client.methods.HttpRequestWrapper;
63  import org.apache.http.client.protocol.HttpClientContext;
64  import org.apache.http.client.utils.DateUtils;
65  import org.apache.http.client.utils.URIUtils;
66  import org.apache.http.conn.routing.HttpRoute;
67  import org.apache.http.impl.execchain.ClientExecChain;
68  import org.apache.http.message.BasicHttpResponse;
69  import org.apache.http.protocol.HTTP;
70  import org.apache.http.protocol.HttpContext;
71  import org.apache.http.protocol.HttpCoreContext;
72  import org.apache.http.util.Args;
73  import org.apache.http.util.VersionInfo;
74  
75  /**
76   * <p>
77   * Request executor in the request execution chain that is responsible for
78   * transparent client-side caching.
79   * </p>
80   * <p>
81   * The current implementation is conditionally
82   * compliant with HTTP/1.1 (meaning all the MUST and MUST NOTs are obeyed),
83   * although quite a lot, though not all, of the SHOULDs and SHOULD NOTs
84   * are obeyed too.
85   * </p>
86   * <p>
87   * Folks that would like to experiment with alternative storage backends
88   * should look at the {@link HttpCacheStorage} interface and the related
89   * package documentation there. You may also be interested in the provided
90   * {@link org.apache.http.impl.client.cache.ehcache.EhcacheHttpCacheStorage
91   * EhCache} and {@link
92   * org.apache.http.impl.client.cache.memcached.MemcachedHttpCacheStorage
93   * memcached} storage backends.
94   * </p>
95   * <p>
96   * Further responsibilities such as communication with the opposite
97   * endpoint is delegated to the next executor in the request execution
98   * chain.
99   * </p>
100  *
101  * @since 4.3
102  */
103 @Contract(threading = ThreadingBehavior.SAFE_CONDITIONAL)
104 public class CachingExec implements ClientExecChain {
105 
106     private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
107 
108     private final AtomicLong cacheHits = new AtomicLong();
109     private final AtomicLong cacheMisses = new AtomicLong();
110     private final AtomicLong cacheUpdates = new AtomicLong();
111 
112     private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
113 
114     private final CacheConfig cacheConfig;
115     private final ClientExecChain backend;
116     private final HttpCache responseCache;
117     private final CacheValidityPolicy validityPolicy;
118     private final CachedHttpResponseGenerator responseGenerator;
119     private final CacheableRequestPolicy cacheableRequestPolicy;
120     private final CachedResponseSuitabilityChecker suitabilityChecker;
121     private final ConditionalRequestBuilder conditionalRequestBuilder;
122     private final ResponseProtocolCompliance responseCompliance;
123     private final RequestProtocolCompliance requestCompliance;
124     private final ResponseCachingPolicy responseCachingPolicy;
125 
126     private final AsynchronousValidator asynchRevalidator;
127 
128     private final Log log = LogFactory.getLog(getClass());
129 
130     public CachingExec(
131             final ClientExecChain backend,
132             final HttpCache cache,
133             final CacheConfig config) {
134         this(backend, cache, config, null);
135     }
136 
137     public CachingExec(
138             final ClientExecChain backend,
139             final HttpCache cache,
140             final CacheConfig config,
141             final AsynchronousValidator asynchRevalidator) {
142         super();
143         Args.notNull(backend, "HTTP backend");
144         Args.notNull(cache, "HttpCache");
145         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
146         this.backend = backend;
147         this.responseCache = cache;
148         this.validityPolicy = new CacheValidityPolicy();
149         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
150         this.cacheableRequestPolicy = new CacheableRequestPolicy();
151         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, this.cacheConfig);
152         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
153         this.responseCompliance = new ResponseProtocolCompliance();
154         this.requestCompliance = new RequestProtocolCompliance(this.cacheConfig.isWeakETagOnPutDeleteAllowed());
155         this.responseCachingPolicy = new ResponseCachingPolicy(
156                 this.cacheConfig.getMaxObjectSize(), this.cacheConfig.isSharedCache(),
157                 this.cacheConfig.isNeverCacheHTTP10ResponsesWithQuery(), this.cacheConfig.is303CachingEnabled());
158         this.asynchRevalidator = asynchRevalidator;
159     }
160 
161     public CachingExec(
162             final ClientExecChain backend,
163             final ResourceFactory resourceFactory,
164             final HttpCacheStorage storage,
165             final CacheConfig config) {
166         this(backend, new BasicHttpCache(resourceFactory, storage, config), config);
167     }
168 
169     public CachingExec(final ClientExecChain backend) {
170         this(backend, new BasicHttpCache(), CacheConfig.DEFAULT);
171     }
172 
173     CachingExec(
174             final ClientExecChain backend,
175             final HttpCache responseCache,
176             final CacheValidityPolicy validityPolicy,
177             final ResponseCachingPolicy responseCachingPolicy,
178             final CachedHttpResponseGenerator responseGenerator,
179             final CacheableRequestPolicy cacheableRequestPolicy,
180             final CachedResponseSuitabilityChecker suitabilityChecker,
181             final ConditionalRequestBuilder conditionalRequestBuilder,
182             final ResponseProtocolCompliance responseCompliance,
183             final RequestProtocolCompliance requestCompliance,
184             final CacheConfig config,
185             final AsynchronousValidator asynchRevalidator) {
186         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
187         this.backend = backend;
188         this.responseCache = responseCache;
189         this.validityPolicy = validityPolicy;
190         this.responseCachingPolicy = responseCachingPolicy;
191         this.responseGenerator = responseGenerator;
192         this.cacheableRequestPolicy = cacheableRequestPolicy;
193         this.suitabilityChecker = suitabilityChecker;
194         this.conditionalRequestBuilder = conditionalRequestBuilder;
195         this.responseCompliance = responseCompliance;
196         this.requestCompliance = requestCompliance;
197         this.asynchRevalidator = asynchRevalidator;
198     }
199 
200     /**
201      * Reports the number of times that the cache successfully responded
202      * to an {@link HttpRequest} without contacting the origin server.
203      * @return the number of cache hits
204      */
205     public long getCacheHits() {
206         return cacheHits.get();
207     }
208 
209     /**
210      * Reports the number of times that the cache contacted the origin
211      * server because it had no appropriate response cached.
212      * @return the number of cache misses
213      */
214     public long getCacheMisses() {
215         return cacheMisses.get();
216     }
217 
218     /**
219      * Reports the number of times that the cache was able to satisfy
220      * a response by revalidating an existing but stale cache entry.
221      * @return the number of cache revalidations
222      */
223     public long getCacheUpdates() {
224         return cacheUpdates.get();
225     }
226 
227     public CloseableHttpResponse execute(
228             final HttpRoute route,
229             final HttpRequestWrapper request) throws IOException, HttpException {
230         return execute(route, request, HttpClientContext.create(), null);
231     }
232 
233     public CloseableHttpResponse execute(
234             final HttpRoute route,
235             final HttpRequestWrapper request,
236             final HttpClientContext context) throws IOException, HttpException {
237         return execute(route, request, context, null);
238     }
239 
240     @Override
241     public CloseableHttpResponse execute(
242             final HttpRoute route,
243             final HttpRequestWrapper request,
244             final HttpClientContext context,
245             final HttpExecutionAware execAware) throws IOException, HttpException {
246 
247         final HttpHost target = context.getTargetHost();
248         final String via = generateViaHeader(request.getOriginal());
249 
250         // default response context
251         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
252 
253         if (clientRequestsOurOptions(request)) {
254             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
255             return Proxies.enhanceResponse(new OptionsHttp11Response());
256         }
257 
258         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(request, context);
259         if (fatalErrorResponse != null) {
260             return Proxies.enhanceResponse(fatalErrorResponse);
261         }
262 
263         requestCompliance.makeRequestCompliant(request);
264         request.addHeader("Via",via);
265 
266         flushEntriesInvalidatedByRequest(context.getTargetHost(), request);
267 
268         if (!cacheableRequestPolicy.isServableFromCache(request)) {
269             log.debug("Request is not servable from cache");
270             return callBackend(route, request, context, execAware);
271         }
272 
273         final HttpCacheEntry entry = satisfyFromCache(target, request);
274         if (entry == null) {
275             log.debug("Cache miss");
276             return handleCacheMiss(route, request, context, execAware);
277         } else {
278             return handleCacheHit(route, request, context, execAware, entry);
279         }
280     }
281 
282     private CloseableHttpResponse handleCacheHit(
283             final HttpRoute route,
284             final HttpRequestWrapper request,
285             final HttpClientContext context,
286             final HttpExecutionAware execAware,
287             final HttpCacheEntry entry) throws IOException, HttpException {
288         final HttpHost target = context.getTargetHost();
289         recordCacheHit(target, request);
290         CloseableHttpResponse out = null;
291         final Date now = getCurrentDate();
292         if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
293             log.debug("Cache hit");
294             out = generateCachedResponse(request, context, entry, now);
295         } else if (!mayCallBackend(request)) {
296             log.debug("Cache entry not suitable but only-if-cached requested");
297             out = generateGatewayTimeout(context);
298         } else if (!(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
299                 && !suitabilityChecker.isConditional(request))) {
300             log.debug("Revalidating cache entry");
301             return revalidateCacheEntry(route, request, context, execAware, entry, now);
302         } else {
303             log.debug("Cache entry not usable; calling backend");
304             return callBackend(route, request, context, execAware);
305         }
306         context.setAttribute(HttpClientContext.HTTP_ROUTE, route);
307         context.setAttribute(HttpCoreContext.HTTP_TARGET_HOST, target);
308         context.setAttribute(HttpCoreContext.HTTP_REQUEST, request);
309         context.setAttribute(HttpCoreContext.HTTP_RESPONSE, out);
310         context.setAttribute(HttpCoreContext.HTTP_REQ_SENT, Boolean.TRUE);
311         return out;
312     }
313 
314     private CloseableHttpResponse revalidateCacheEntry(
315             final HttpRoute route,
316             final HttpRequestWrapper request,
317             final HttpClientContext context,
318             final HttpExecutionAware execAware,
319             final HttpCacheEntry entry,
320             final Date now) throws HttpException {
321 
322         try {
323             if (asynchRevalidator != null
324                 && !staleResponseNotAllowed(request, entry, now)
325                 && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
326                 log.trace("Serving stale with asynchronous revalidation");
327                 final CloseableHttpResponse resp = generateCachedResponse(request, context, entry, now);
328                 asynchRevalidator.revalidateCacheEntry(this, route, request, context, execAware, entry);
329                 return resp;
330             }
331             return revalidateCacheEntry(route, request, context, execAware, entry);
332         } catch (final IOException ioex) {
333             return handleRevalidationFailure(request, context, entry, now);
334         }
335     }
336 
337     private CloseableHttpResponse handleCacheMiss(
338             final HttpRoute route,
339             final HttpRequestWrapper request,
340             final HttpClientContext context,
341             final HttpExecutionAware execAware) throws IOException, HttpException {
342         final HttpHost target = context.getTargetHost();
343         recordCacheMiss(target, request);
344 
345         if (!mayCallBackend(request)) {
346             return Proxies.enhanceResponse(
347                     new BasicHttpResponse(
348                             HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
349         }
350 
351         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
352         if (variants != null && !variants.isEmpty()) {
353             return negotiateResponseFromVariants(route, request, context,
354                     execAware, variants);
355         }
356 
357         return callBackend(route, request, context, execAware);
358     }
359 
360     private HttpCacheEntry satisfyFromCache(
361             final HttpHost target, final HttpRequestWrapper request) {
362         HttpCacheEntry entry = null;
363         try {
364             entry = responseCache.getCacheEntry(target, request);
365         } catch (final IOException ioe) {
366             log.warn("Unable to retrieve entries from cache", ioe);
367         }
368         return entry;
369     }
370 
371     private HttpResponse getFatallyNoncompliantResponse(
372             final HttpRequestWrapper request,
373             final HttpContext context) {
374         HttpResponse fatalErrorResponse = null;
375         final List<RequestProtocolError> fatalError = requestCompliance.requestIsFatallyNonCompliant(request);
376 
377         for (final RequestProtocolError error : fatalError) {
378             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
379             fatalErrorResponse = requestCompliance.getErrorForRequest(error);
380         }
381         return fatalErrorResponse;
382     }
383 
384     private Map<String, Variant> getExistingCacheVariants(
385             final HttpHost target,
386             final HttpRequestWrapper request) {
387         Map<String,Variant> variants = null;
388         try {
389             variants = responseCache.getVariantCacheEntriesWithEtags(target, request);
390         } catch (final IOException ioe) {
391             log.warn("Unable to retrieve variant entries from cache", ioe);
392         }
393         return variants;
394     }
395 
396     private void recordCacheMiss(final HttpHost target, final HttpRequestWrapper request) {
397         cacheMisses.getAndIncrement();
398         if (log.isTraceEnabled()) {
399             final RequestLine rl = request.getRequestLine();
400             log.trace("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
401         }
402     }
403 
404     private void recordCacheHit(final HttpHost target, final HttpRequestWrapper request) {
405         cacheHits.getAndIncrement();
406         if (log.isTraceEnabled()) {
407             final RequestLine rl = request.getRequestLine();
408             log.trace("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
409         }
410     }
411 
412     private void recordCacheUpdate(final HttpContext context) {
413         cacheUpdates.getAndIncrement();
414         setResponseStatus(context, CacheResponseStatus.VALIDATED);
415     }
416 
417     private void flushEntriesInvalidatedByRequest(
418             final HttpHost target,
419             final HttpRequestWrapper request) {
420         try {
421             responseCache.flushInvalidatedCacheEntriesFor(target, request);
422         } catch (final IOException ioe) {
423             log.warn("Unable to flush invalidated entries from cache", ioe);
424         }
425     }
426 
427     private CloseableHttpResponse generateCachedResponse(final HttpRequestWrapper request,
428             final HttpContext context, final HttpCacheEntry entry, final Date now) {
429         final CloseableHttpResponse cachedResponse;
430         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
431                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
432             cachedResponse = responseGenerator.generateNotModifiedResponse(entry);
433         } else {
434             cachedResponse = responseGenerator.generateResponse(request, entry);
435         }
436         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
437         if (validityPolicy.getStalenessSecs(entry, now) > 0L) {
438             cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\"");
439         }
440         return cachedResponse;
441     }
442 
443     private CloseableHttpResponse handleRevalidationFailure(
444             final HttpRequestWrapper request,
445             final HttpContext context,
446             final HttpCacheEntry entry,
447             final Date now) {
448         if (staleResponseNotAllowed(request, entry, now)) {
449             return generateGatewayTimeout(context);
450         } else {
451             return unvalidatedCacheHit(request, context, entry);
452         }
453     }
454 
455     private CloseableHttpResponse generateGatewayTimeout(
456             final HttpContext context) {
457         setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
458         return Proxies.enhanceResponse(new BasicHttpResponse(
459                 HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT,
460                 "Gateway Timeout"));
461     }
462 
463     private CloseableHttpResponse unvalidatedCacheHit(
464             final HttpRequestWrapper request,
465             final HttpContext context,
466             final HttpCacheEntry entry) {
467         final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(request, entry);
468         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
469         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
470         return cachedResponse;
471     }
472 
473     private boolean staleResponseNotAllowed(
474             final HttpRequestWrapper request,
475             final HttpCacheEntry entry,
476             final Date now) {
477         return validityPolicy.mustRevalidate(entry)
478             || (cacheConfig.isSharedCache() && validityPolicy.proxyRevalidate(entry))
479             || explicitFreshnessRequest(request, entry, now);
480     }
481 
482     private boolean mayCallBackend(final HttpRequestWrapper request) {
483         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
484             for (final HeaderElement elt : h.getElements()) {
485                 if ("only-if-cached".equals(elt.getName())) {
486                     log.trace("Request marked only-if-cached");
487                     return false;
488                 }
489             }
490         }
491         return true;
492     }
493 
494     private boolean explicitFreshnessRequest(
495             final HttpRequestWrapper request,
496             final HttpCacheEntry entry,
497             final Date now) {
498         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
499             for(final HeaderElement elt : h.getElements()) {
500                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
501                     try {
502                         final int maxstale = Integer.parseInt(elt.getValue());
503                         final long age = validityPolicy.getCurrentAgeSecs(entry, now);
504                         final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry);
505                         if (age - lifetime > maxstale) {
506                             return true;
507                         }
508                     } catch (final NumberFormatException nfe) {
509                         return true;
510                     }
511                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
512                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
513                     return true;
514                 }
515             }
516         }
517         return false;
518     }
519 
520     private String generateViaHeader(final HttpMessage msg) {
521 
522         final ProtocolVersion pv = msg.getProtocolVersion();
523         final String existingEntry = viaHeaders.get(pv);
524         if (existingEntry != null) {
525             return existingEntry;
526         }
527 
528         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
529         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
530 
531         String value;
532         final int major = pv.getMajor();
533         final int minor = pv.getMinor();
534         if ("http".equalsIgnoreCase(pv.getProtocol())) {
535             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", major, minor,
536                     release);
537         } else {
538             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), major,
539                     minor, release);
540         }
541         viaHeaders.put(pv, value);
542 
543         return value;
544     }
545 
546     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
547         if (context != null) {
548             context.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
549         }
550     }
551 
552     /**
553      * Reports whether this {@code CachingHttpClient} implementation
554      * supports byte-range requests as specified by the {@code Range}
555      * and {@code Content-Range} headers.
556      * @return {@code true} if byte-range requests are supported
557      */
558     public boolean supportsRangeAndContentRangeHeaders() {
559         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
560     }
561 
562     Date getCurrentDate() {
563         return new Date();
564     }
565 
566     boolean clientRequestsOurOptions(final HttpRequest request) {
567         final RequestLine line = request.getRequestLine();
568 
569         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
570             return false;
571         }
572 
573         if (!"*".equals(line.getUri())) {
574             return false;
575         }
576 
577         if (!"0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue())) {
578             return false;
579         }
580 
581         return true;
582     }
583 
584     CloseableHttpResponse callBackend(
585             final HttpRoute route,
586             final HttpRequestWrapper request,
587             final HttpClientContext context,
588             final HttpExecutionAware execAware) throws IOException, HttpException  {
589 
590         final Date requestDate = getCurrentDate();
591 
592         log.trace("Calling the backend");
593         final CloseableHttpResponse backendResponse = backend.execute(route, request, context, execAware);
594         try {
595             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
596             return handleBackendResponse(request, context, requestDate, getCurrentDate(),
597                     backendResponse);
598         } catch (final IOException ex) {
599             backendResponse.close();
600             throw ex;
601         } catch (final RuntimeException ex) {
602             backendResponse.close();
603             throw ex;
604         }
605     }
606 
607     private boolean revalidationResponseIsTooOld(final HttpResponse backendResponse,
608             final HttpCacheEntry cacheEntry) {
609         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
610         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
611         if (entryDateHeader != null && responseDateHeader != null) {
612             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
613             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
614             if (entryDate == null || respDate == null) {
615                 // either backend response or cached entry did not have a valid
616                 // Date header, so we can't tell if they are out of order
617                 // according to the origin clock; thus we can skip the
618                 // unconditional retry recommended in 13.2.6 of RFC 2616.
619                 return false;
620             }
621             if (respDate.before(entryDate)) {
622                 return true;
623             }
624         }
625         return false;
626     }
627 
628     CloseableHttpResponse negotiateResponseFromVariants(
629             final HttpRoute route,
630             final HttpRequestWrapper request,
631             final HttpClientContext context,
632             final HttpExecutionAware execAware,
633             final Map<String, Variant> variants) throws IOException, HttpException {
634         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder
635             .buildConditionalRequestFromVariants(request, variants);
636 
637         final Date requestDate = getCurrentDate();
638         final CloseableHttpResponse backendResponse = backend.execute(
639                 route, conditionalRequest, context, execAware);
640         try {
641             final Date responseDate = getCurrentDate();
642 
643             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
644 
645             if (backendResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
646                 return handleBackendResponse(request, context, requestDate, responseDate,
647                         backendResponse);
648             }
649 
650             final Header resultEtagHeader = backendResponse.getFirstHeader(HeaderConstants.ETAG);
651             if (resultEtagHeader == null) {
652                 log.warn("304 response did not contain ETag");
653                 IOUtils.consume(backendResponse.getEntity());
654                 backendResponse.close();
655                 return callBackend(route, request, context, execAware);
656             }
657 
658             final String resultEtag = resultEtagHeader.getValue();
659             final Variant matchingVariant = variants.get(resultEtag);
660             if (matchingVariant == null) {
661                 log.debug("304 response did not contain ETag matching one sent in If-None-Match");
662                 IOUtils.consume(backendResponse.getEntity());
663                 backendResponse.close();
664                 return callBackend(route, request, context, execAware);
665             }
666 
667             final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
668 
669             if (revalidationResponseIsTooOld(backendResponse, matchedEntry)) {
670                 IOUtils.consume(backendResponse.getEntity());
671                 backendResponse.close();
672                 return retryRequestUnconditionally(route, request, context, execAware, matchedEntry);
673             }
674 
675             recordCacheUpdate(context);
676 
677             final HttpCacheEntry responseEntry = getUpdatedVariantEntry(
678                 context.getTargetHost(), conditionalRequest, requestDate, responseDate,
679                     backendResponse, matchingVariant, matchedEntry);
680             backendResponse.close();
681 
682             final CloseableHttpResponse resp = responseGenerator.generateResponse(request, responseEntry);
683             tryToUpdateVariantMap(context.getTargetHost(), request, matchingVariant);
684 
685             if (shouldSendNotModifiedResponse(request, responseEntry)) {
686                 return responseGenerator.generateNotModifiedResponse(responseEntry);
687             }
688             return resp;
689         } catch (final IOException ex) {
690             backendResponse.close();
691             throw ex;
692         } catch (final RuntimeException ex) {
693             backendResponse.close();
694             throw ex;
695         }
696     }
697 
698     private CloseableHttpResponse retryRequestUnconditionally(
699             final HttpRoute route,
700             final HttpRequestWrapper request,
701             final HttpClientContext context,
702             final HttpExecutionAware execAware,
703             final HttpCacheEntry matchedEntry) throws IOException, HttpException {
704         final HttpRequestWrapper unconditional = conditionalRequestBuilder
705             .buildUnconditionalRequest(request, matchedEntry);
706         return callBackend(route, unconditional, context, execAware);
707     }
708 
709     private HttpCacheEntry getUpdatedVariantEntry(
710             final HttpHost target,
711             final HttpRequestWrapper conditionalRequest,
712             final Date requestDate,
713             final Date responseDate,
714             final CloseableHttpResponse backendResponse,
715             final Variant matchingVariant,
716             final HttpCacheEntry matchedEntry) throws IOException {
717         HttpCacheEntry responseEntry = matchedEntry;
718         try {
719             responseEntry = responseCache.updateVariantCacheEntry(target, conditionalRequest,
720                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
721         } catch (final IOException ioe) {
722             log.warn("Could not update cache entry", ioe);
723         } finally {
724             backendResponse.close();
725         }
726         return responseEntry;
727     }
728 
729     private void tryToUpdateVariantMap(
730             final HttpHost target,
731             final HttpRequestWrapper request,
732             final Variant matchingVariant) {
733         try {
734             responseCache.reuseVariantEntryFor(target, request, matchingVariant);
735         } catch (final IOException ioe) {
736             log.warn("Could not update cache entry to reuse variant", ioe);
737         }
738     }
739 
740     private boolean shouldSendNotModifiedResponse(
741             final HttpRequestWrapper request,
742             final HttpCacheEntry responseEntry) {
743         return (suitabilityChecker.isConditional(request)
744                 && suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
745     }
746 
747     CloseableHttpResponse revalidateCacheEntry(
748             final HttpRoute route,
749             final HttpRequestWrapper request,
750             final HttpClientContext context,
751             final HttpExecutionAware execAware,
752             final HttpCacheEntry cacheEntry) throws IOException, HttpException {
753 
754         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
755         final URI uri = conditionalRequest.getURI();
756         if (uri != null) {
757             try {
758                 conditionalRequest.setURI(URIUtils.rewriteURIForRoute(uri, route));
759             } catch (final URISyntaxException ex) {
760                 throw new ProtocolException("Invalid URI: " + uri, ex);
761             }
762         }
763 
764         Date requestDate = getCurrentDate();
765         CloseableHttpResponse backendResponse = backend.execute(
766                 route, conditionalRequest, context, execAware);
767         Date responseDate = getCurrentDate();
768 
769         if (revalidationResponseIsTooOld(backendResponse, cacheEntry)) {
770             backendResponse.close();
771             final HttpRequestWrapper unconditional = conditionalRequestBuilder
772                 .buildUnconditionalRequest(request, cacheEntry);
773             requestDate = getCurrentDate();
774             backendResponse = backend.execute(route, unconditional, context, execAware);
775             responseDate = getCurrentDate();
776         }
777 
778         backendResponse.addHeader(HeaderConstants.VIA, generateViaHeader(backendResponse));
779 
780         final int statusCode = backendResponse.getStatusLine().getStatusCode();
781         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
782             recordCacheUpdate(context);
783         }
784 
785         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
786             final HttpCacheEntry updatedEntry = responseCache.updateCacheEntry(
787                     context.getTargetHost(), request, cacheEntry,
788                     backendResponse, requestDate, responseDate);
789             if (suitabilityChecker.isConditional(request)
790                     && suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
791                 return responseGenerator
792                         .generateNotModifiedResponse(updatedEntry);
793             }
794             return responseGenerator.generateResponse(request, updatedEntry);
795         }
796 
797         if (staleIfErrorAppliesTo(statusCode)
798             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
799             && validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
800             try {
801                 final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(request, cacheEntry);
802                 cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
803                 return cachedResponse;
804             } finally {
805                 backendResponse.close();
806             }
807         }
808         return handleBackendResponse(conditionalRequest, context, requestDate, responseDate,
809                 backendResponse);
810     }
811 
812     private boolean staleIfErrorAppliesTo(final int statusCode) {
813         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
814                 || statusCode == HttpStatus.SC_BAD_GATEWAY
815                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
816                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
817     }
818 
819     CloseableHttpResponse handleBackendResponse(
820             final HttpRequestWrapper request,
821             final HttpClientContext context,
822             final Date requestDate,
823             final Date responseDate,
824             final CloseableHttpResponse backendResponse) throws IOException {
825 
826         log.trace("Handling Backend response");
827         responseCompliance.ensureProtocolCompliance(request, backendResponse);
828 
829         final HttpHost target = context.getTargetHost();
830         final boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse);
831         responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
832         if (cacheable && !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
833             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
834             return responseCache.cacheAndReturnResponse(target, request,
835                     backendResponse, requestDate, responseDate);
836         }
837         if (!cacheable) {
838             try {
839                 responseCache.flushCacheEntriesFor(target, request);
840             } catch (final IOException ioe) {
841                 log.warn("Unable to flush invalid cache entries", ioe);
842             }
843         }
844         return backendResponse;
845     }
846 
847     /**
848      * For 304 Not modified responses, adds a "Last-Modified" header with the
849      * value of the "If-Modified-Since" header passed in the request. This
850      * header is required to be able to reuse match the cache entry for
851      * subsequent requests but as defined in http specifications it is not
852      * included in 304 responses by backend servers. This header will not be
853      * included in the resulting response.
854      */
855     private void storeRequestIfModifiedSinceFor304Response(
856             final HttpRequest request, final HttpResponse backendResponse) {
857         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
858             final Header h = request.getFirstHeader("If-Modified-Since");
859             if (h != null) {
860                 backendResponse.addHeader("Last-Modified", h.getValue());
861             }
862         }
863     }
864 
865     private boolean alreadyHaveNewerCacheEntry(final HttpHost target, final HttpRequestWrapper request,
866             final HttpResponse backendResponse) {
867         HttpCacheEntry existing = null;
868         try {
869             existing = responseCache.getCacheEntry(target, request);
870         } catch (final IOException ioe) {
871             // nop
872         }
873         if (existing == null) {
874             return false;
875         }
876         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
877         if (entryDateHeader == null) {
878             return false;
879         }
880         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
881         if (responseDateHeader == null) {
882             return false;
883         }
884         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
885         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
886         if (entryDate == null || responseDate == null) {
887             return false;
888         }
889         return responseDate.before(entryDate);
890     }
891 
892 }