View Javadoc
1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.net.URI;
31  import java.net.URISyntaxException;
32  import java.util.Date;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.http.Header;
41  import org.apache.http.HeaderElement;
42  import org.apache.http.HttpException;
43  import org.apache.http.HttpHost;
44  import org.apache.http.HttpMessage;
45  import org.apache.http.HttpRequest;
46  import org.apache.http.HttpResponse;
47  import org.apache.http.HttpStatus;
48  import org.apache.http.HttpVersion;
49  import org.apache.http.ProtocolException;
50  import org.apache.http.ProtocolVersion;
51  import org.apache.http.RequestLine;
52  import org.apache.http.annotation.Contract;
53  import org.apache.http.annotation.ThreadingBehavior;
54  import org.apache.http.client.cache.CacheResponseStatus;
55  import org.apache.http.client.cache.HeaderConstants;
56  import org.apache.http.client.cache.HttpCacheContext;
57  import org.apache.http.client.cache.HttpCacheEntry;
58  import org.apache.http.client.cache.HttpCacheStorage;
59  import org.apache.http.client.cache.ResourceFactory;
60  import org.apache.http.client.methods.CloseableHttpResponse;
61  import org.apache.http.client.methods.HttpExecutionAware;
62  import org.apache.http.client.methods.HttpRequestWrapper;
63  import org.apache.http.client.protocol.HttpClientContext;
64  import org.apache.http.client.utils.DateUtils;
65  import org.apache.http.client.utils.URIUtils;
66  import org.apache.http.conn.routing.HttpRoute;
67  import org.apache.http.impl.execchain.ClientExecChain;
68  import org.apache.http.message.BasicHttpResponse;
69  import org.apache.http.protocol.HTTP;
70  import org.apache.http.protocol.HttpContext;
71  import org.apache.http.protocol.HttpCoreContext;
72  import org.apache.http.util.Args;
73  import org.apache.http.util.VersionInfo;
74  
75  /**
76   * <p>
77   * Request executor in the request execution chain that is responsible for
78   * transparent client-side caching.
79   * </p>
80   * <p>
81   * The current implementation is conditionally
82   * compliant with HTTP/1.1 (meaning all the MUST and MUST NOTs are obeyed),
83   * although quite a lot, though not all, of the SHOULDs and SHOULD NOTs
84   * are obeyed too.
85   * </p>
86   * <p>
87   * Folks that would like to experiment with alternative storage backends
88   * should look at the {@link HttpCacheStorage} interface and the related
89   * package documentation there. You may also be interested in the provided
90   * {@link org.apache.http.impl.client.cache.ehcache.EhcacheHttpCacheStorage
91   * EhCache} and {@link
92   * org.apache.http.impl.client.cache.memcached.MemcachedHttpCacheStorage
93   * memcached} storage backends.
94   * </p>
95   * <p>
96   * Further responsibilities such as communication with the opposite
97   * endpoint is delegated to the next executor in the request execution
98   * chain.
99   * </p>
100  *
101  * @since 4.3
102  */
103 @Contract(threading = ThreadingBehavior.SAFE_CONDITIONAL)
104 public class CachingExec implements ClientExecChain {
105 
106     private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
107 
108     private final AtomicLong cacheHits = new AtomicLong();
109     private final AtomicLong cacheMisses = new AtomicLong();
110     private final AtomicLong cacheUpdates = new AtomicLong();
111 
112     private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
113 
114     private final CacheConfig cacheConfig;
115     private final ClientExecChain backend;
116     private final HttpCache responseCache;
117     private final CacheValidityPolicy validityPolicy;
118     private final CachedHttpResponseGenerator responseGenerator;
119     private final CacheableRequestPolicy cacheableRequestPolicy;
120     private final CachedResponseSuitabilityChecker suitabilityChecker;
121     private final ConditionalRequestBuilder conditionalRequestBuilder;
122     private final ResponseProtocolCompliance responseCompliance;
123     private final RequestProtocolCompliance requestCompliance;
124     private final ResponseCachingPolicy responseCachingPolicy;
125 
126     private final AsynchronousValidator asynchRevalidator;
127 
128     private final Log log = LogFactory.getLog(getClass());
129 
130     public CachingExec(
131             final ClientExecChain backend,
132             final HttpCache cache,
133             final CacheConfig config) {
134         this(backend, cache, config, null);
135     }
136 
137     public CachingExec(
138             final ClientExecChain backend,
139             final HttpCache cache,
140             final CacheConfig config,
141             final AsynchronousValidator asynchRevalidator) {
142         super();
143         Args.notNull(backend, "HTTP backend");
144         Args.notNull(cache, "HttpCache");
145         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
146         this.backend = backend;
147         this.responseCache = cache;
148         this.validityPolicy = new CacheValidityPolicy();
149         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
150         this.cacheableRequestPolicy = new CacheableRequestPolicy();
151         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, this.cacheConfig);
152         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
153         this.responseCompliance = new ResponseProtocolCompliance();
154         this.requestCompliance = new RequestProtocolCompliance(this.cacheConfig.isWeakETagOnPutDeleteAllowed());
155         this.responseCachingPolicy = new ResponseCachingPolicy(
156                 this.cacheConfig.getMaxObjectSize(), this.cacheConfig.isSharedCache(),
157                 this.cacheConfig.isNeverCacheHTTP10ResponsesWithQuery(), this.cacheConfig.is303CachingEnabled());
158         this.asynchRevalidator = asynchRevalidator;
159     }
160 
161     public CachingExec(
162             final ClientExecChain backend,
163             final ResourceFactory resourceFactory,
164             final HttpCacheStorage storage,
165             final CacheConfig config) {
166         this(backend, new BasicHttpCache(resourceFactory, storage, config), config);
167     }
168 
169     public CachingExec(final ClientExecChain backend) {
170         this(backend, new BasicHttpCache(), CacheConfig.DEFAULT);
171     }
172 
173     CachingExec(
174             final ClientExecChain backend,
175             final HttpCache responseCache,
176             final CacheValidityPolicy validityPolicy,
177             final ResponseCachingPolicy responseCachingPolicy,
178             final CachedHttpResponseGenerator responseGenerator,
179             final CacheableRequestPolicy cacheableRequestPolicy,
180             final CachedResponseSuitabilityChecker suitabilityChecker,
181             final ConditionalRequestBuilder conditionalRequestBuilder,
182             final ResponseProtocolCompliance responseCompliance,
183             final RequestProtocolCompliance requestCompliance,
184             final CacheConfig config,
185             final AsynchronousValidator asynchRevalidator) {
186         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
187         this.backend = backend;
188         this.responseCache = responseCache;
189         this.validityPolicy = validityPolicy;
190         this.responseCachingPolicy = responseCachingPolicy;
191         this.responseGenerator = responseGenerator;
192         this.cacheableRequestPolicy = cacheableRequestPolicy;
193         this.suitabilityChecker = suitabilityChecker;
194         this.conditionalRequestBuilder = conditionalRequestBuilder;
195         this.responseCompliance = responseCompliance;
196         this.requestCompliance = requestCompliance;
197         this.asynchRevalidator = asynchRevalidator;
198     }
199 
200     /**
201      * Reports the number of times that the cache successfully responded
202      * to an {@link HttpRequest} without contacting the origin server.
203      * @return the number of cache hits
204      */
205     public long getCacheHits() {
206         return cacheHits.get();
207     }
208 
209     /**
210      * Reports the number of times that the cache contacted the origin
211      * server because it had no appropriate response cached.
212      * @return the number of cache misses
213      */
214     public long getCacheMisses() {
215         return cacheMisses.get();
216     }
217 
218     /**
219      * Reports the number of times that the cache was able to satisfy
220      * a response by revalidating an existing but stale cache entry.
221      * @return the number of cache revalidations
222      */
223     public long getCacheUpdates() {
224         return cacheUpdates.get();
225     }
226 
227     public CloseableHttpResponse execute(
228             final HttpRoute route,
229             final HttpRequestWrapper request) throws IOException, HttpException {
230         return execute(route, request, HttpClientContext.create(), null);
231     }
232 
233     public CloseableHttpResponse execute(
234             final HttpRoute route,
235             final HttpRequestWrapper request,
236             final HttpClientContext context) throws IOException, HttpException {
237         return execute(route, request, context, null);
238     }
239 
240     @Override
241     public CloseableHttpResponse execute(
242             final HttpRoute route,
243             final HttpRequestWrapper request,
244             final HttpClientContext context,
245             final HttpExecutionAware execAware) throws IOException, HttpException {
246 
247         final HttpHost target = context.getTargetHost();
248         final String via = generateViaHeader(request.getOriginal());
249 
250         // default response context
251         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
252 
253         if (clientRequestsOurOptions(request)) {
254             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
255             return Proxies.enhanceResponse(new OptionsHttp11Response());
256         }
257 
258         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(request, context);
259         if (fatalErrorResponse != null) {
260             return Proxies.enhanceResponse(fatalErrorResponse);
261         }
262 
263         requestCompliance.makeRequestCompliant(request);
264         request.addHeader("Via",via);
265 
266         if (!cacheableRequestPolicy.isServableFromCache(request)) {
267             log.debug("Request is not servable from cache");
268             flushEntriesInvalidatedByRequest(context.getTargetHost(), request);
269             return callBackend(route, request, context, execAware);
270         }
271 
272         final HttpCacheEntry entry = satisfyFromCache(target, request);
273         if (entry == null) {
274             log.debug("Cache miss");
275             return handleCacheMiss(route, request, context, execAware);
276         } else {
277             return handleCacheHit(route, request, context, execAware, entry);
278         }
279     }
280 
281     private CloseableHttpResponse handleCacheHit(
282             final HttpRoute route,
283             final HttpRequestWrapper request,
284             final HttpClientContext context,
285             final HttpExecutionAware execAware,
286             final HttpCacheEntry entry) throws IOException, HttpException {
287         final HttpHost target = context.getTargetHost();
288         recordCacheHit(target, request);
289         CloseableHttpResponse out = null;
290         final Date now = getCurrentDate();
291         if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
292             log.debug("Cache hit");
293             out = generateCachedResponse(request, context, entry, now);
294         } else if (!mayCallBackend(request)) {
295             log.debug("Cache entry not suitable but only-if-cached requested");
296             out = generateGatewayTimeout(context);
297         } else if (!(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
298                 && !suitabilityChecker.isConditional(request))) {
299             log.debug("Revalidating cache entry");
300             return revalidateCacheEntry(route, request, context, execAware, entry, now);
301         } else {
302             log.debug("Cache entry not usable; calling backend");
303             return callBackend(route, request, context, execAware);
304         }
305         context.setAttribute(HttpClientContext.HTTP_ROUTE, route);
306         context.setAttribute(HttpCoreContext.HTTP_TARGET_HOST, target);
307         context.setAttribute(HttpCoreContext.HTTP_REQUEST, request);
308         context.setAttribute(HttpCoreContext.HTTP_RESPONSE, out);
309         context.setAttribute(HttpCoreContext.HTTP_REQ_SENT, Boolean.TRUE);
310         return out;
311     }
312 
313     private CloseableHttpResponse revalidateCacheEntry(
314             final HttpRoute route,
315             final HttpRequestWrapper request,
316             final HttpClientContext context,
317             final HttpExecutionAware execAware,
318             final HttpCacheEntry entry,
319             final Date now) throws HttpException {
320 
321         try {
322             if (asynchRevalidator != null
323                 && !staleResponseNotAllowed(request, entry, now)
324                 && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
325                 log.trace("Serving stale with asynchronous revalidation");
326                 final CloseableHttpResponse resp = generateCachedResponse(request, context, entry, now);
327                 asynchRevalidator.revalidateCacheEntry(this, route, request, context, execAware, entry);
328                 return resp;
329             }
330             return revalidateCacheEntry(route, request, context, execAware, entry);
331         } catch (final IOException ioex) {
332             return handleRevalidationFailure(request, context, entry, now);
333         }
334     }
335 
336     private CloseableHttpResponse handleCacheMiss(
337             final HttpRoute route,
338             final HttpRequestWrapper request,
339             final HttpClientContext context,
340             final HttpExecutionAware execAware) throws IOException, HttpException {
341         final HttpHost target = context.getTargetHost();
342         recordCacheMiss(target, request);
343 
344         if (!mayCallBackend(request)) {
345             return Proxies.enhanceResponse(
346                     new BasicHttpResponse(
347                             HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
348         }
349 
350         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
351         if (variants != null && !variants.isEmpty()) {
352             return negotiateResponseFromVariants(route, request, context,
353                     execAware, variants);
354         }
355 
356         return callBackend(route, request, context, execAware);
357     }
358 
359     private HttpCacheEntry satisfyFromCache(
360             final HttpHost target, final HttpRequestWrapper request) {
361         HttpCacheEntry entry = null;
362         try {
363             entry = responseCache.getCacheEntry(target, request);
364         } catch (final IOException ioe) {
365             log.warn("Unable to retrieve entries from cache", ioe);
366         }
367         return entry;
368     }
369 
370     private HttpResponse getFatallyNoncompliantResponse(
371             final HttpRequestWrapper request,
372             final HttpContext context) {
373         HttpResponse fatalErrorResponse = null;
374         final List<RequestProtocolError> fatalError = requestCompliance.requestIsFatallyNonCompliant(request);
375 
376         for (final RequestProtocolError error : fatalError) {
377             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
378             fatalErrorResponse = requestCompliance.getErrorForRequest(error);
379         }
380         return fatalErrorResponse;
381     }
382 
383     private Map<String, Variant> getExistingCacheVariants(
384             final HttpHost target,
385             final HttpRequestWrapper request) {
386         Map<String,Variant> variants = null;
387         try {
388             variants = responseCache.getVariantCacheEntriesWithEtags(target, request);
389         } catch (final IOException ioe) {
390             log.warn("Unable to retrieve variant entries from cache", ioe);
391         }
392         return variants;
393     }
394 
395     private void recordCacheMiss(final HttpHost target, final HttpRequestWrapper request) {
396         cacheMisses.getAndIncrement();
397         if (log.isTraceEnabled()) {
398             final RequestLine rl = request.getRequestLine();
399             log.trace("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
400         }
401     }
402 
403     private void recordCacheHit(final HttpHost target, final HttpRequestWrapper request) {
404         cacheHits.getAndIncrement();
405         if (log.isTraceEnabled()) {
406             final RequestLine rl = request.getRequestLine();
407             log.trace("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
408         }
409     }
410 
411     private void recordCacheUpdate(final HttpContext context) {
412         cacheUpdates.getAndIncrement();
413         setResponseStatus(context, CacheResponseStatus.VALIDATED);
414     }
415 
416     private void flushEntriesInvalidatedByRequest(
417             final HttpHost target,
418             final HttpRequestWrapper request) {
419         try {
420             responseCache.flushInvalidatedCacheEntriesFor(target, request);
421         } catch (final IOException ioe) {
422             log.warn("Unable to flush invalidated entries from cache", ioe);
423         }
424     }
425 
426     private CloseableHttpResponse generateCachedResponse(final HttpRequestWrapper request,
427             final HttpContext context, final HttpCacheEntry entry, final Date now) {
428         final CloseableHttpResponse cachedResponse;
429         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
430                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
431             cachedResponse = responseGenerator.generateNotModifiedResponse(entry);
432         } else {
433             cachedResponse = responseGenerator.generateResponse(request, entry);
434         }
435         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
436         if (validityPolicy.getStalenessSecs(entry, now) > 0L) {
437             cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\"");
438         }
439         return cachedResponse;
440     }
441 
442     private CloseableHttpResponse handleRevalidationFailure(
443             final HttpRequestWrapper request,
444             final HttpContext context,
445             final HttpCacheEntry entry,
446             final Date now) {
447         if (staleResponseNotAllowed(request, entry, now)) {
448             return generateGatewayTimeout(context);
449         } else {
450             return unvalidatedCacheHit(request, context, entry);
451         }
452     }
453 
454     private CloseableHttpResponse generateGatewayTimeout(
455             final HttpContext context) {
456         setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
457         return Proxies.enhanceResponse(new BasicHttpResponse(
458                 HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT,
459                 "Gateway Timeout"));
460     }
461 
462     private CloseableHttpResponse unvalidatedCacheHit(
463             final HttpRequestWrapper request,
464             final HttpContext context,
465             final HttpCacheEntry entry) {
466         final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(request, entry);
467         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
468         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
469         return cachedResponse;
470     }
471 
472     private boolean staleResponseNotAllowed(
473             final HttpRequestWrapper request,
474             final HttpCacheEntry entry,
475             final Date now) {
476         return validityPolicy.mustRevalidate(entry)
477             || (cacheConfig.isSharedCache() && validityPolicy.proxyRevalidate(entry))
478             || explicitFreshnessRequest(request, entry, now);
479     }
480 
481     private boolean mayCallBackend(final HttpRequestWrapper request) {
482         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
483             for (final HeaderElement elt : h.getElements()) {
484                 if ("only-if-cached".equals(elt.getName())) {
485                     log.trace("Request marked only-if-cached");
486                     return false;
487                 }
488             }
489         }
490         return true;
491     }
492 
493     private boolean explicitFreshnessRequest(
494             final HttpRequestWrapper request,
495             final HttpCacheEntry entry,
496             final Date now) {
497         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
498             for(final HeaderElement elt : h.getElements()) {
499                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
500                     try {
501                         final int maxstale = Integer.parseInt(elt.getValue());
502                         final long age = validityPolicy.getCurrentAgeSecs(entry, now);
503                         final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry);
504                         if (age - lifetime > maxstale) {
505                             return true;
506                         }
507                     } catch (final NumberFormatException nfe) {
508                         return true;
509                     }
510                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
511                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
512                     return true;
513                 }
514             }
515         }
516         return false;
517     }
518 
519     private String generateViaHeader(final HttpMessage msg) {
520 
521         final ProtocolVersion pv = msg.getProtocolVersion();
522         final String existingEntry = viaHeaders.get(pv);
523         if (existingEntry != null) {
524             return existingEntry;
525         }
526 
527         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
528         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
529 
530         final String value;
531         final int major = pv.getMajor();
532         final int minor = pv.getMinor();
533         if ("http".equalsIgnoreCase(pv.getProtocol())) {
534             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", major, minor,
535                     release);
536         } else {
537             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), major,
538                     minor, release);
539         }
540         viaHeaders.put(pv, value);
541 
542         return value;
543     }
544 
545     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
546         if (context != null) {
547             context.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
548         }
549     }
550 
551     /**
552      * Reports whether this {@code CachingHttpClient} implementation
553      * supports byte-range requests as specified by the {@code Range}
554      * and {@code Content-Range} headers.
555      * @return {@code true} if byte-range requests are supported
556      */
557     public boolean supportsRangeAndContentRangeHeaders() {
558         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
559     }
560 
561     Date getCurrentDate() {
562         return new Date();
563     }
564 
565     boolean clientRequestsOurOptions(final HttpRequest request) {
566         final RequestLine line = request.getRequestLine();
567 
568         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
569             return false;
570         }
571 
572         if (!"*".equals(line.getUri())) {
573             return false;
574         }
575 
576         if (!"0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue())) {
577             return false;
578         }
579 
580         return true;
581     }
582 
583     CloseableHttpResponse callBackend(
584             final HttpRoute route,
585             final HttpRequestWrapper request,
586             final HttpClientContext context,
587             final HttpExecutionAware execAware) throws IOException, HttpException  {
588 
589         final Date requestDate = getCurrentDate();
590 
591         log.trace("Calling the backend");
592         final CloseableHttpResponse backendResponse = backend.execute(route, request, context, execAware);
593         try {
594             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
595             return handleBackendResponse(request, context, requestDate, getCurrentDate(),
596                     backendResponse);
597         } catch (final IOException ex) {
598             backendResponse.close();
599             throw ex;
600         } catch (final RuntimeException ex) {
601             backendResponse.close();
602             throw ex;
603         }
604     }
605 
606     private boolean revalidationResponseIsTooOld(final HttpResponse backendResponse,
607             final HttpCacheEntry cacheEntry) {
608         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
609         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
610         if (entryDateHeader != null && responseDateHeader != null) {
611             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
612             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
613             if (entryDate == null || respDate == null) {
614                 // either backend response or cached entry did not have a valid
615                 // Date header, so we can't tell if they are out of order
616                 // according to the origin clock; thus we can skip the
617                 // unconditional retry recommended in 13.2.6 of RFC 2616.
618                 return false;
619             }
620             if (respDate.before(entryDate)) {
621                 return true;
622             }
623         }
624         return false;
625     }
626 
627     CloseableHttpResponse negotiateResponseFromVariants(
628             final HttpRoute route,
629             final HttpRequestWrapper request,
630             final HttpClientContext context,
631             final HttpExecutionAware execAware,
632             final Map<String, Variant> variants) throws IOException, HttpException {
633         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder
634             .buildConditionalRequestFromVariants(request, variants);
635 
636         final Date requestDate = getCurrentDate();
637         final CloseableHttpResponse backendResponse = backend.execute(
638                 route, conditionalRequest, context, execAware);
639         try {
640             final Date responseDate = getCurrentDate();
641 
642             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
643 
644             if (backendResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
645                 return handleBackendResponse(request, context, requestDate, responseDate,
646                         backendResponse);
647             }
648 
649             final Header resultEtagHeader = backendResponse.getFirstHeader(HeaderConstants.ETAG);
650             if (resultEtagHeader == null) {
651                 log.warn("304 response did not contain ETag");
652                 IOUtils.consume(backendResponse.getEntity());
653                 backendResponse.close();
654                 return callBackend(route, request, context, execAware);
655             }
656 
657             final String resultEtag = resultEtagHeader.getValue();
658             final Variant matchingVariant = variants.get(resultEtag);
659             if (matchingVariant == null) {
660                 log.debug("304 response did not contain ETag matching one sent in If-None-Match");
661                 IOUtils.consume(backendResponse.getEntity());
662                 backendResponse.close();
663                 return callBackend(route, request, context, execAware);
664             }
665 
666             final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
667 
668             if (revalidationResponseIsTooOld(backendResponse, matchedEntry)) {
669                 IOUtils.consume(backendResponse.getEntity());
670                 backendResponse.close();
671                 return retryRequestUnconditionally(route, request, context, execAware, matchedEntry);
672             }
673 
674             recordCacheUpdate(context);
675 
676             final HttpCacheEntry responseEntry = getUpdatedVariantEntry(
677                 context.getTargetHost(), conditionalRequest, requestDate, responseDate,
678                     backendResponse, matchingVariant, matchedEntry);
679             backendResponse.close();
680 
681             final CloseableHttpResponse resp = responseGenerator.generateResponse(request, responseEntry);
682             tryToUpdateVariantMap(context.getTargetHost(), request, matchingVariant);
683 
684             if (shouldSendNotModifiedResponse(request, responseEntry)) {
685                 return responseGenerator.generateNotModifiedResponse(responseEntry);
686             }
687             return resp;
688         } catch (final IOException ex) {
689             backendResponse.close();
690             throw ex;
691         } catch (final RuntimeException ex) {
692             backendResponse.close();
693             throw ex;
694         }
695     }
696 
697     private CloseableHttpResponse retryRequestUnconditionally(
698             final HttpRoute route,
699             final HttpRequestWrapper request,
700             final HttpClientContext context,
701             final HttpExecutionAware execAware,
702             final HttpCacheEntry matchedEntry) throws IOException, HttpException {
703         final HttpRequestWrapper unconditional = conditionalRequestBuilder
704             .buildUnconditionalRequest(request, matchedEntry);
705         return callBackend(route, unconditional, context, execAware);
706     }
707 
708     private HttpCacheEntry getUpdatedVariantEntry(
709             final HttpHost target,
710             final HttpRequestWrapper conditionalRequest,
711             final Date requestDate,
712             final Date responseDate,
713             final CloseableHttpResponse backendResponse,
714             final Variant matchingVariant,
715             final HttpCacheEntry matchedEntry) throws IOException {
716         HttpCacheEntry responseEntry = matchedEntry;
717         try {
718             responseEntry = responseCache.updateVariantCacheEntry(target, conditionalRequest,
719                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
720         } catch (final IOException ioe) {
721             log.warn("Could not update cache entry", ioe);
722         } finally {
723             backendResponse.close();
724         }
725         return responseEntry;
726     }
727 
728     private void tryToUpdateVariantMap(
729             final HttpHost target,
730             final HttpRequestWrapper request,
731             final Variant matchingVariant) {
732         try {
733             responseCache.reuseVariantEntryFor(target, request, matchingVariant);
734         } catch (final IOException ioe) {
735             log.warn("Could not update cache entry to reuse variant", ioe);
736         }
737     }
738 
739     private boolean shouldSendNotModifiedResponse(
740             final HttpRequestWrapper request,
741             final HttpCacheEntry responseEntry) {
742         return (suitabilityChecker.isConditional(request)
743                 && suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
744     }
745 
746     CloseableHttpResponse revalidateCacheEntry(
747             final HttpRoute route,
748             final HttpRequestWrapper request,
749             final HttpClientContext context,
750             final HttpExecutionAware execAware,
751             final HttpCacheEntry cacheEntry) throws IOException, HttpException {
752 
753         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
754         final URI uri = conditionalRequest.getURI();
755         if (uri != null) {
756             try {
757                 conditionalRequest.setURI(URIUtils.rewriteURIForRoute(uri, route));
758             } catch (final URISyntaxException ex) {
759                 throw new ProtocolException("Invalid URI: " + uri, ex);
760             }
761         }
762 
763         Date requestDate = getCurrentDate();
764         CloseableHttpResponse backendResponse = backend.execute(
765                 route, conditionalRequest, context, execAware);
766         Date responseDate = getCurrentDate();
767 
768         if (revalidationResponseIsTooOld(backendResponse, cacheEntry)) {
769             backendResponse.close();
770             final HttpRequestWrapper unconditional = conditionalRequestBuilder
771                 .buildUnconditionalRequest(request, cacheEntry);
772             requestDate = getCurrentDate();
773             backendResponse = backend.execute(route, unconditional, context, execAware);
774             responseDate = getCurrentDate();
775         }
776 
777         backendResponse.addHeader(HeaderConstants.VIA, generateViaHeader(backendResponse));
778 
779         final int statusCode = backendResponse.getStatusLine().getStatusCode();
780         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
781             recordCacheUpdate(context);
782         }
783 
784         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
785             final HttpCacheEntry updatedEntry = responseCache.updateCacheEntry(
786                     context.getTargetHost(), request, cacheEntry,
787                     backendResponse, requestDate, responseDate);
788             if (suitabilityChecker.isConditional(request)
789                     && suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
790                 return responseGenerator
791                         .generateNotModifiedResponse(updatedEntry);
792             }
793             return responseGenerator.generateResponse(request, updatedEntry);
794         }
795 
796         if (staleIfErrorAppliesTo(statusCode)
797             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
798             && validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
799             try {
800                 final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(request, cacheEntry);
801                 cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
802                 return cachedResponse;
803             } finally {
804                 backendResponse.close();
805             }
806         }
807         return handleBackendResponse(conditionalRequest, context, requestDate, responseDate,
808                 backendResponse);
809     }
810 
811     private boolean staleIfErrorAppliesTo(final int statusCode) {
812         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
813                 || statusCode == HttpStatus.SC_BAD_GATEWAY
814                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
815                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
816     }
817 
818     CloseableHttpResponse handleBackendResponse(
819             final HttpRequestWrapper request,
820             final HttpClientContext context,
821             final Date requestDate,
822             final Date responseDate,
823             final CloseableHttpResponse backendResponse) throws IOException {
824 
825         log.trace("Handling Backend response");
826         responseCompliance.ensureProtocolCompliance(request, backendResponse);
827 
828         final HttpHost target = context.getTargetHost();
829         final boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse);
830         responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
831         if (cacheable && !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
832             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
833             return responseCache.cacheAndReturnResponse(target, request,
834                     backendResponse, requestDate, responseDate);
835         }
836         if (!cacheable) {
837             try {
838                 responseCache.flushCacheEntriesFor(target, request);
839             } catch (final IOException ioe) {
840                 log.warn("Unable to flush invalid cache entries", ioe);
841             }
842         }
843         return backendResponse;
844     }
845 
846     /**
847      * For 304 Not modified responses, adds a "Last-Modified" header with the
848      * value of the "If-Modified-Since" header passed in the request. This
849      * header is required to be able to reuse match the cache entry for
850      * subsequent requests but as defined in http specifications it is not
851      * included in 304 responses by backend servers. This header will not be
852      * included in the resulting response.
853      */
854     private void storeRequestIfModifiedSinceFor304Response(
855             final HttpRequest request, final HttpResponse backendResponse) {
856         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
857             final Header h = request.getFirstHeader("If-Modified-Since");
858             if (h != null) {
859                 backendResponse.addHeader("Last-Modified", h.getValue());
860             }
861         }
862     }
863 
864     private boolean alreadyHaveNewerCacheEntry(final HttpHost target, final HttpRequestWrapper request,
865             final HttpResponse backendResponse) {
866         HttpCacheEntry existing = null;
867         try {
868             existing = responseCache.getCacheEntry(target, request);
869         } catch (final IOException ioe) {
870             // nop
871         }
872         if (existing == null) {
873             return false;
874         }
875         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
876         if (entryDateHeader == null) {
877             return false;
878         }
879         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
880         if (responseDateHeader == null) {
881             return false;
882         }
883         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
884         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
885         if (entryDate == null || responseDate == null) {
886             return false;
887         }
888         return responseDate.before(entryDate);
889     }
890 
891 }