View Javadoc

1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.util.Date;
31  import java.util.HashMap;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.concurrent.atomic.AtomicLong;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.http.Header;
39  import org.apache.http.HeaderElement;
40  import org.apache.http.HttpException;
41  import org.apache.http.HttpHost;
42  import org.apache.http.HttpMessage;
43  import org.apache.http.HttpRequest;
44  import org.apache.http.HttpResponse;
45  import org.apache.http.HttpStatus;
46  import org.apache.http.HttpVersion;
47  import org.apache.http.ProtocolVersion;
48  import org.apache.http.RequestLine;
49  import org.apache.http.annotation.ThreadSafe;
50  import org.apache.http.client.cache.CacheResponseStatus;
51  import org.apache.http.client.cache.HeaderConstants;
52  import org.apache.http.client.cache.HttpCacheContext;
53  import org.apache.http.client.cache.HttpCacheEntry;
54  import org.apache.http.client.cache.HttpCacheStorage;
55  import org.apache.http.client.cache.ResourceFactory;
56  import org.apache.http.client.methods.CloseableHttpResponse;
57  import org.apache.http.client.methods.HttpExecutionAware;
58  import org.apache.http.client.methods.HttpRequestWrapper;
59  import org.apache.http.client.protocol.HttpClientContext;
60  import org.apache.http.client.utils.DateUtils;
61  import org.apache.http.conn.routing.HttpRoute;
62  import org.apache.http.impl.execchain.ClientExecChain;
63  import org.apache.http.message.BasicHttpResponse;
64  import org.apache.http.protocol.HTTP;
65  import org.apache.http.protocol.HttpContext;
66  import org.apache.http.util.Args;
67  import org.apache.http.util.VersionInfo;
68  
69  /**
70   * Request executor in the request execution chain that is responsible for
71   * transparent client-side caching. The current implementation is conditionally
72   * compliant with HTTP/1.1 (meaning all the MUST and MUST NOTs are obeyed),
73   * although quite a lot, though not all, of the SHOULDs and SHOULD NOTs
74   * are obeyed too.
75   * <p/>
76   * Folks that would like to experiment with alternative storage backends
77   * should look at the {@link HttpCacheStorage} interface and the related
78   * package documentation there. You may also be interested in the provided
79   * {@link org.apache.http.impl.client.cache.ehcache.EhcacheHttpCacheStorage
80   * EhCache} and {@link
81   * org.apache.http.impl.client.cache.memcached.MemcachedHttpCacheStorage
82   * memcached} storage backends.
83   * <p/>
84   * Further responsibilities such as communication with the opposite
85   * endpoint is delegated to the next executor in the request execution
86   * chain.
87   *
88   * @since 4.3
89   */
90  @ThreadSafe // So long as the responseCache implementation is threadsafe
91  public class CachingExec implements ClientExecChain {
92  
93      private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
94  
95      private final AtomicLong cacheHits = new AtomicLong();
96      private final AtomicLong cacheMisses = new AtomicLong();
97      private final AtomicLong cacheUpdates = new AtomicLong();
98  
99      private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
100 
101     private final CacheConfig cacheConfig;
102     private final ClientExecChain backend;
103     private final HttpCache responseCache;
104     private final CacheValidityPolicy validityPolicy;
105     private final CachedHttpResponseGenerator responseGenerator;
106     private final CacheableRequestPolicy cacheableRequestPolicy;
107     private final CachedResponseSuitabilityChecker suitabilityChecker;
108     private final ConditionalRequestBuilder conditionalRequestBuilder;
109     private final ResponseProtocolCompliance responseCompliance;
110     private final RequestProtocolCompliance requestCompliance;
111     private final ResponseCachingPolicy responseCachingPolicy;
112 
113     private final AsynchronousValidator asynchRevalidator;
114 
115     private final Log log = LogFactory.getLog(getClass());
116 
117     public CachingExec(
118             final ClientExecChain backend,
119             final HttpCache cache,
120             final CacheConfig config) {
121         this(backend, cache, config, null);
122     }
123 
124     public CachingExec(
125             final ClientExecChain backend,
126             final HttpCache cache,
127             final CacheConfig config,
128             final AsynchronousValidator asynchRevalidator) {
129         super();
130         Args.notNull(backend, "HTTP backend");
131         Args.notNull(cache, "HttpCache");
132         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
133         this.backend = backend;
134         this.responseCache = cache;
135         this.validityPolicy = new CacheValidityPolicy();
136         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
137         this.cacheableRequestPolicy = new CacheableRequestPolicy();
138         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, config);
139         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
140         this.responseCompliance = new ResponseProtocolCompliance();
141         this.requestCompliance = new RequestProtocolCompliance(config.isWeakETagOnPutDeleteAllowed());
142         this.responseCachingPolicy = new ResponseCachingPolicy(
143                 this.cacheConfig.getMaxObjectSize(), this.cacheConfig.isSharedCache(),
144                 this.cacheConfig.isNeverCacheHTTP10ResponsesWithQuery(), this.cacheConfig.is303CachingEnabled());
145         this.asynchRevalidator = asynchRevalidator;
146     }
147 
148     public CachingExec(
149             final ClientExecChain backend,
150             final ResourceFactory resourceFactory,
151             final HttpCacheStorage storage,
152             final CacheConfig config) {
153         this(backend, new BasicHttpCache(resourceFactory, storage, config), config);
154     }
155 
156     public CachingExec(final ClientExecChain backend) {
157         this(backend, new BasicHttpCache(), CacheConfig.DEFAULT);
158     }
159 
160     CachingExec(
161             final ClientExecChain backend,
162             final HttpCache responseCache,
163             final CacheValidityPolicy validityPolicy,
164             final ResponseCachingPolicy responseCachingPolicy,
165             final CachedHttpResponseGenerator responseGenerator,
166             final CacheableRequestPolicy cacheableRequestPolicy,
167             final CachedResponseSuitabilityChecker suitabilityChecker,
168             final ConditionalRequestBuilder conditionalRequestBuilder,
169             final ResponseProtocolCompliance responseCompliance,
170             final RequestProtocolCompliance requestCompliance,
171             final CacheConfig config,
172             final AsynchronousValidator asynchRevalidator) {
173         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
174         this.backend = backend;
175         this.responseCache = responseCache;
176         this.validityPolicy = validityPolicy;
177         this.responseCachingPolicy = responseCachingPolicy;
178         this.responseGenerator = responseGenerator;
179         this.cacheableRequestPolicy = cacheableRequestPolicy;
180         this.suitabilityChecker = suitabilityChecker;
181         this.conditionalRequestBuilder = conditionalRequestBuilder;
182         this.responseCompliance = responseCompliance;
183         this.requestCompliance = requestCompliance;
184         this.asynchRevalidator = asynchRevalidator;
185     }
186 
187     /**
188      * Reports the number of times that the cache successfully responded
189      * to an {@link HttpRequest} without contacting the origin server.
190      * @return the number of cache hits
191      */
192     public long getCacheHits() {
193         return cacheHits.get();
194     }
195 
196     /**
197      * Reports the number of times that the cache contacted the origin
198      * server because it had no appropriate response cached.
199      * @return the number of cache misses
200      */
201     public long getCacheMisses() {
202         return cacheMisses.get();
203     }
204 
205     /**
206      * Reports the number of times that the cache was able to satisfy
207      * a response by revalidating an existing but stale cache entry.
208      * @return the number of cache revalidations
209      */
210     public long getCacheUpdates() {
211         return cacheUpdates.get();
212     }
213 
214     public CloseableHttpResponse execute(
215             final HttpRoute route,
216             final HttpRequestWrapper request) throws IOException, HttpException {
217         return execute(route, request, HttpClientContext.create(), null);
218     }
219 
220     public CloseableHttpResponse execute(
221             final HttpRoute route,
222             final HttpRequestWrapper request,
223             final HttpClientContext context) throws IOException, HttpException {
224         return execute(route, request, context, null);
225     }
226 
227     public CloseableHttpResponse execute(
228             final HttpRoute route,
229             final HttpRequestWrapper request,
230             final HttpClientContext context,
231             final HttpExecutionAware execAware) throws IOException, HttpException {
232 
233         final HttpHost target = context.getTargetHost();
234         final String via = generateViaHeader(request.getOriginal());
235 
236         // default response context
237         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
238 
239         if (clientRequestsOurOptions(request)) {
240             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
241             return Proxies.enhanceResponse(new OptionsHttp11Response());
242         }
243 
244         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(request, context);
245         if (fatalErrorResponse != null) {
246             return Proxies.enhanceResponse(fatalErrorResponse);
247         }
248 
249         requestCompliance.makeRequestCompliant(request);
250         request.addHeader("Via",via);
251 
252         flushEntriesInvalidatedByRequest(context.getTargetHost(), request);
253 
254         if (!cacheableRequestPolicy.isServableFromCache(request)) {
255             log.debug("Request is not servable from cache");
256             return callBackend(route, request, context, execAware);
257         }
258 
259         final HttpCacheEntry entry = satisfyFromCache(target, request);
260         if (entry == null) {
261             log.debug("Cache miss");
262             return handleCacheMiss(route, request, context, execAware);
263         } else {
264             return handleCacheHit(route, request, context, execAware, entry);
265         }
266     }
267 
268     private CloseableHttpResponse handleCacheHit(
269             final HttpRoute route,
270             final HttpRequestWrapper request,
271             final HttpClientContext context,
272             final HttpExecutionAware execAware,
273             final HttpCacheEntry entry) throws IOException, HttpException {
274         final HttpHost target = context.getTargetHost();
275         recordCacheHit(target, request);
276         CloseableHttpResponse out = null;
277         final Date now = getCurrentDate();
278         if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
279             log.debug("Cache hit");
280             out = generateCachedResponse(request, context, entry, now);
281         } else if (!mayCallBackend(request)) {
282             log.debug("Cache entry not suitable but only-if-cached requested");
283             out = generateGatewayTimeout(context);
284         } else if (!(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
285                 && !suitabilityChecker.isConditional(request))) {
286             log.debug("Revalidating cache entry");
287             return revalidateCacheEntry(route, request, context, execAware, entry, now);
288         } else {
289             log.debug("Cache entry not usable; calling backend");
290             return callBackend(route, request, context, execAware);
291         }
292         context.setAttribute(HttpClientContext.HTTP_ROUTE, route);
293         context.setAttribute(HttpClientContext.HTTP_TARGET_HOST, target);
294         context.setAttribute(HttpClientContext.HTTP_REQUEST, request);
295         context.setAttribute(HttpClientContext.HTTP_RESPONSE, out);
296         context.setAttribute(HttpClientContext.HTTP_REQ_SENT, Boolean.TRUE);
297         return out;
298     }
299 
300     private CloseableHttpResponse revalidateCacheEntry(
301             final HttpRoute route,
302             final HttpRequestWrapper request,
303             final HttpClientContext context,
304             final HttpExecutionAware execAware,
305             final HttpCacheEntry entry,
306             final Date now) throws HttpException {
307 
308         try {
309             if (asynchRevalidator != null
310                 && !staleResponseNotAllowed(request, entry, now)
311                 && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
312                 log.trace("Serving stale with asynchronous revalidation");
313                 final CloseableHttpResponse resp = generateCachedResponse(request, context, entry, now);
314                 asynchRevalidator.revalidateCacheEntry(this, route, request, context, execAware, entry);
315                 return resp;
316             }
317             return revalidateCacheEntry(route, request, context, execAware, entry);
318         } catch (final IOException ioex) {
319             return handleRevalidationFailure(request, context, entry, now);
320         }
321     }
322 
323     private CloseableHttpResponse handleCacheMiss(
324             final HttpRoute route,
325             final HttpRequestWrapper request,
326             final HttpClientContext context,
327             final HttpExecutionAware execAware) throws IOException, HttpException {
328         final HttpHost target = context.getTargetHost();
329         recordCacheMiss(target, request);
330 
331         if (!mayCallBackend(request)) {
332             return Proxies.enhanceResponse(
333                     new BasicHttpResponse(
334                             HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
335         }
336 
337         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
338         if (variants != null && variants.size() > 0) {
339             return negotiateResponseFromVariants(route, request, context,
340                     execAware, variants);
341         }
342 
343         return callBackend(route, request, context, execAware);
344     }
345 
346     private HttpCacheEntry satisfyFromCache(
347             final HttpHost target, final HttpRequestWrapper request) {
348         HttpCacheEntry entry = null;
349         try {
350             entry = responseCache.getCacheEntry(target, request);
351         } catch (final IOException ioe) {
352             log.warn("Unable to retrieve entries from cache", ioe);
353         }
354         return entry;
355     }
356 
357     private HttpResponse getFatallyNoncompliantResponse(
358             final HttpRequestWrapper request,
359             final HttpContext context) {
360         HttpResponse fatalErrorResponse = null;
361         final List<RequestProtocolError> fatalError = requestCompliance.requestIsFatallyNonCompliant(request);
362 
363         for (final RequestProtocolError error : fatalError) {
364             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
365             fatalErrorResponse = requestCompliance.getErrorForRequest(error);
366         }
367         return fatalErrorResponse;
368     }
369 
370     private Map<String, Variant> getExistingCacheVariants(
371             final HttpHost target,
372             final HttpRequestWrapper request) {
373         Map<String,Variant> variants = null;
374         try {
375             variants = responseCache.getVariantCacheEntriesWithEtags(target, request);
376         } catch (final IOException ioe) {
377             log.warn("Unable to retrieve variant entries from cache", ioe);
378         }
379         return variants;
380     }
381 
382     private void recordCacheMiss(final HttpHost target, final HttpRequestWrapper request) {
383         cacheMisses.getAndIncrement();
384         if (log.isTraceEnabled()) {
385             final RequestLine rl = request.getRequestLine();
386             log.trace("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
387         }
388     }
389 
390     private void recordCacheHit(final HttpHost target, final HttpRequestWrapper request) {
391         cacheHits.getAndIncrement();
392         if (log.isTraceEnabled()) {
393             final RequestLine rl = request.getRequestLine();
394             log.trace("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
395         }
396     }
397 
398     private void recordCacheUpdate(final HttpContext context) {
399         cacheUpdates.getAndIncrement();
400         setResponseStatus(context, CacheResponseStatus.VALIDATED);
401     }
402 
403     private void flushEntriesInvalidatedByRequest(
404             final HttpHost target,
405             final HttpRequestWrapper request) {
406         try {
407             responseCache.flushInvalidatedCacheEntriesFor(target, request);
408         } catch (final IOException ioe) {
409             log.warn("Unable to flush invalidated entries from cache", ioe);
410         }
411     }
412 
413     private CloseableHttpResponse generateCachedResponse(final HttpRequestWrapper request,
414             final HttpContext context, final HttpCacheEntry entry, final Date now) {
415         final CloseableHttpResponse cachedResponse;
416         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
417                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
418             cachedResponse = responseGenerator.generateNotModifiedResponse(entry);
419         } else {
420             cachedResponse = responseGenerator.generateResponse(entry);
421         }
422         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
423         if (validityPolicy.getStalenessSecs(entry, now) > 0L) {
424             cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\"");
425         }
426         return cachedResponse;
427     }
428 
429     private CloseableHttpResponse handleRevalidationFailure(
430             final HttpRequestWrapper request,
431             final HttpContext context,
432             final HttpCacheEntry entry,
433             final Date now) {
434         if (staleResponseNotAllowed(request, entry, now)) {
435             return generateGatewayTimeout(context);
436         } else {
437             return unvalidatedCacheHit(context, entry);
438         }
439     }
440 
441     private CloseableHttpResponse generateGatewayTimeout(
442             final HttpContext context) {
443         setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
444         return Proxies.enhanceResponse(new BasicHttpResponse(
445                 HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT,
446                 "Gateway Timeout"));
447     }
448 
449     private CloseableHttpResponse unvalidatedCacheHit(
450             final HttpContext context, final HttpCacheEntry entry) {
451         final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(entry);
452         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
453         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
454         return cachedResponse;
455     }
456 
457     private boolean staleResponseNotAllowed(
458             final HttpRequestWrapper request,
459             final HttpCacheEntry entry,
460             final Date now) {
461         return validityPolicy.mustRevalidate(entry)
462             || (cacheConfig.isSharedCache() && validityPolicy.proxyRevalidate(entry))
463             || explicitFreshnessRequest(request, entry, now);
464     }
465 
466     private boolean mayCallBackend(final HttpRequestWrapper request) {
467         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
468             for (final HeaderElement elt : h.getElements()) {
469                 if ("only-if-cached".equals(elt.getName())) {
470                     log.trace("Request marked only-if-cached");
471                     return false;
472                 }
473             }
474         }
475         return true;
476     }
477 
478     private boolean explicitFreshnessRequest(
479             final HttpRequestWrapper request,
480             final HttpCacheEntry entry,
481             final Date now) {
482         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
483             for(final HeaderElement elt : h.getElements()) {
484                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
485                     try {
486                         final int maxstale = Integer.parseInt(elt.getValue());
487                         final long age = validityPolicy.getCurrentAgeSecs(entry, now);
488                         final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry);
489                         if (age - lifetime > maxstale) {
490                             return true;
491                         }
492                     } catch (final NumberFormatException nfe) {
493                         return true;
494                     }
495                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
496                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
497                     return true;
498                 }
499             }
500         }
501         return false;
502     }
503 
504     private String generateViaHeader(final HttpMessage msg) {
505 
506         final ProtocolVersion pv = msg.getProtocolVersion();
507         final String existingEntry = viaHeaders.get(pv);
508         if (existingEntry != null) {
509             return existingEntry;
510         }
511 
512         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
513         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
514 
515         String value;
516         if ("http".equalsIgnoreCase(pv.getProtocol())) {
517             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getMajor(), pv.getMinor(),
518                     release);
519         } else {
520             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), pv.getMajor(),
521                     pv.getMinor(), release);
522         }
523         viaHeaders.put(pv, value);
524 
525         return value;
526     }
527 
528     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
529         if (context != null) {
530             context.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
531         }
532     }
533 
534     /**
535      * Reports whether this {@code CachingHttpClient} implementation
536      * supports byte-range requests as specified by the {@code Range}
537      * and {@code Content-Range} headers.
538      * @return {@code true} if byte-range requests are supported
539      */
540     public boolean supportsRangeAndContentRangeHeaders() {
541         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
542     }
543 
544     Date getCurrentDate() {
545         return new Date();
546     }
547 
548     boolean clientRequestsOurOptions(final HttpRequest request) {
549         final RequestLine line = request.getRequestLine();
550 
551         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
552             return false;
553         }
554 
555         if (!"*".equals(line.getUri())) {
556             return false;
557         }
558 
559         if (!"0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue())) {
560             return false;
561         }
562 
563         return true;
564     }
565 
566     CloseableHttpResponse callBackend(
567             final HttpRoute route,
568             final HttpRequestWrapper request,
569             final HttpClientContext context,
570             final HttpExecutionAware execAware) throws IOException, HttpException  {
571 
572         final Date requestDate = getCurrentDate();
573 
574         log.trace("Calling the backend");
575         final CloseableHttpResponse backendResponse = backend.execute(route, request, context, execAware);
576         try {
577             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
578             return handleBackendResponse(route, request, context, execAware,
579                     requestDate, getCurrentDate(), backendResponse);
580         } catch (final IOException ex) {
581             backendResponse.close();
582             throw ex;
583         } catch (final RuntimeException ex) {
584             backendResponse.close();
585             throw ex;
586         }
587     }
588 
589     private boolean revalidationResponseIsTooOld(final HttpResponse backendResponse,
590             final HttpCacheEntry cacheEntry) {
591         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
592         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
593         if (entryDateHeader != null && responseDateHeader != null) {
594             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
595             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
596             if (entryDate == null || respDate == null) {
597                 // either backend response or cached entry did not have a valid
598                 // Date header, so we can't tell if they are out of order
599                 // according to the origin clock; thus we can skip the
600                 // unconditional retry recommended in 13.2.6 of RFC 2616.
601                 return false;
602             }
603             if (respDate.before(entryDate)) {
604                 return true;
605             }
606         }
607         return false;
608     }
609 
610     CloseableHttpResponse negotiateResponseFromVariants(
611             final HttpRoute route,
612             final HttpRequestWrapper request,
613             final HttpClientContext context,
614             final HttpExecutionAware execAware,
615             final Map<String, Variant> variants) throws IOException, HttpException {
616         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder
617             .buildConditionalRequestFromVariants(request, variants);
618 
619         final Date requestDate = getCurrentDate();
620         final CloseableHttpResponse backendResponse = backend.execute(
621                 route, conditionalRequest, context, execAware);
622         try {
623             final Date responseDate = getCurrentDate();
624 
625             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
626 
627             if (backendResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
628                 return handleBackendResponse(
629                         route, request, context, execAware,
630                         requestDate, responseDate, backendResponse);
631             }
632 
633             final Header resultEtagHeader = backendResponse.getFirstHeader(HeaderConstants.ETAG);
634             if (resultEtagHeader == null) {
635                 log.warn("304 response did not contain ETag");
636                 IOUtils.consume(backendResponse.getEntity());
637                 backendResponse.close();
638                 return callBackend(route, request, context, execAware);
639             }
640 
641             final String resultEtag = resultEtagHeader.getValue();
642             final Variant matchingVariant = variants.get(resultEtag);
643             if (matchingVariant == null) {
644                 log.debug("304 response did not contain ETag matching one sent in If-None-Match");
645                 IOUtils.consume(backendResponse.getEntity());
646                 backendResponse.close();
647                 return callBackend(route, request, context, execAware);
648             }
649 
650             final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
651 
652             if (revalidationResponseIsTooOld(backendResponse, matchedEntry)) {
653                 IOUtils.consume(backendResponse.getEntity());
654                 backendResponse.close();
655                 return retryRequestUnconditionally(route, request, context, execAware, matchedEntry);
656             }
657 
658             recordCacheUpdate(context);
659 
660             final HttpCacheEntry responseEntry = getUpdatedVariantEntry(
661                 context.getTargetHost(), conditionalRequest, requestDate, responseDate,
662                     backendResponse, matchingVariant, matchedEntry);
663             backendResponse.close();
664 
665             final CloseableHttpResponse resp = responseGenerator.generateResponse(responseEntry);
666             tryToUpdateVariantMap(context.getTargetHost(), request, matchingVariant);
667 
668             if (shouldSendNotModifiedResponse(request, responseEntry)) {
669                 return responseGenerator.generateNotModifiedResponse(responseEntry);
670             }
671             return resp;
672         } catch (final IOException ex) {
673             backendResponse.close();
674             throw ex;
675         } catch (final RuntimeException ex) {
676             backendResponse.close();
677             throw ex;
678         }
679     }
680 
681     private CloseableHttpResponse retryRequestUnconditionally(
682             final HttpRoute route,
683             final HttpRequestWrapper request,
684             final HttpClientContext context,
685             final HttpExecutionAware execAware,
686             final HttpCacheEntry matchedEntry) throws IOException, HttpException {
687         final HttpRequestWrapper unconditional = conditionalRequestBuilder
688             .buildUnconditionalRequest(request, matchedEntry);
689         return callBackend(route, unconditional, context, execAware);
690     }
691 
692     private HttpCacheEntry getUpdatedVariantEntry(
693             final HttpHost target,
694             final HttpRequestWrapper conditionalRequest,
695             final Date requestDate,
696             final Date responseDate,
697             final CloseableHttpResponse backendResponse,
698             final Variant matchingVariant,
699             final HttpCacheEntry matchedEntry) throws IOException {
700         HttpCacheEntry responseEntry = matchedEntry;
701         try {
702             responseEntry = responseCache.updateVariantCacheEntry(target, conditionalRequest,
703                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
704         } catch (final IOException ioe) {
705             log.warn("Could not update cache entry", ioe);
706         } finally {
707             backendResponse.close();
708         }
709         return responseEntry;
710     }
711 
712     private void tryToUpdateVariantMap(
713             final HttpHost target,
714             final HttpRequestWrapper request,
715             final Variant matchingVariant) {
716         try {
717             responseCache.reuseVariantEntryFor(target, request, matchingVariant);
718         } catch (final IOException ioe) {
719             log.warn("Could not update cache entry to reuse variant", ioe);
720         }
721     }
722 
723     private boolean shouldSendNotModifiedResponse(
724             final HttpRequestWrapper request,
725             final HttpCacheEntry responseEntry) {
726         return (suitabilityChecker.isConditional(request)
727                 && suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
728     }
729 
730     CloseableHttpResponse revalidateCacheEntry(
731             final HttpRoute route,
732             final HttpRequestWrapper request,
733             final HttpClientContext context,
734             final HttpExecutionAware execAware,
735             final HttpCacheEntry cacheEntry) throws IOException, HttpException {
736 
737         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
738 
739         Date requestDate = getCurrentDate();
740         CloseableHttpResponse backendResponse = backend.execute(
741                 route, conditionalRequest, context, execAware);
742         Date responseDate = getCurrentDate();
743 
744         if (revalidationResponseIsTooOld(backendResponse, cacheEntry)) {
745             backendResponse.close();
746             final HttpRequestWrapper unconditional = conditionalRequestBuilder
747                 .buildUnconditionalRequest(request, cacheEntry);
748             requestDate = getCurrentDate();
749             backendResponse = backend.execute(route, unconditional, context, execAware);
750             responseDate = getCurrentDate();
751         }
752 
753         backendResponse.addHeader(HeaderConstants.VIA, generateViaHeader(backendResponse));
754 
755         final int statusCode = backendResponse.getStatusLine().getStatusCode();
756         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
757             recordCacheUpdate(context);
758         }
759 
760         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
761             final HttpCacheEntry updatedEntry = responseCache.updateCacheEntry(
762                     context.getTargetHost(), request, cacheEntry,
763                     backendResponse, requestDate, responseDate);
764             if (suitabilityChecker.isConditional(request)
765                     && suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
766                 return responseGenerator
767                         .generateNotModifiedResponse(updatedEntry);
768             }
769             return responseGenerator.generateResponse(updatedEntry);
770         }
771 
772         if (staleIfErrorAppliesTo(statusCode)
773             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
774             && validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
775             try {
776                 final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(cacheEntry);
777                 cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
778                 return cachedResponse;
779             } finally {
780                 backendResponse.close();
781             }
782         }
783         return handleBackendResponse(
784                 route, conditionalRequest, context, execAware,
785                 requestDate, responseDate, backendResponse);
786     }
787 
788     private boolean staleIfErrorAppliesTo(final int statusCode) {
789         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
790                 || statusCode == HttpStatus.SC_BAD_GATEWAY
791                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
792                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
793     }
794 
795     CloseableHttpResponse handleBackendResponse(
796             final HttpRoute route,
797             final HttpRequestWrapper request,
798             final HttpClientContext context,
799             final HttpExecutionAware execAware,
800             final Date requestDate,
801             final Date responseDate,
802             final CloseableHttpResponse backendResponse) throws IOException {
803 
804         log.trace("Handling Backend response");
805         responseCompliance.ensureProtocolCompliance(request, backendResponse);
806 
807         final HttpHost target = context.getTargetHost();
808         final boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse);
809         responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
810         if (cacheable && !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
811             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
812             return responseCache.cacheAndReturnResponse(target, request,
813                     backendResponse, requestDate, responseDate);
814         }
815         if (!cacheable) {
816             try {
817                 responseCache.flushCacheEntriesFor(target, request);
818             } catch (final IOException ioe) {
819                 log.warn("Unable to flush invalid cache entries", ioe);
820             }
821         }
822         return backendResponse;
823     }
824 
825     /**
826      * For 304 Not modified responses, adds a "Last-Modified" header with the
827      * value of the "If-Modified-Since" header passed in the request. This
828      * header is required to be able to reuse match the cache entry for
829      * subsequent requests but as defined in http specifications it is not
830      * included in 304 responses by backend servers. This header will not be
831      * included in the resulting response.
832      */
833     private void storeRequestIfModifiedSinceFor304Response(
834             final HttpRequest request, final HttpResponse backendResponse) {
835         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
836             final Header h = request.getFirstHeader("If-Modified-Since");
837             if (h != null) {
838                 backendResponse.addHeader("Last-Modified", h.getValue());
839             }
840         }
841     }
842 
843     private boolean alreadyHaveNewerCacheEntry(final HttpHost target, final HttpRequestWrapper request,
844             final HttpResponse backendResponse) {
845         HttpCacheEntry existing = null;
846         try {
847             existing = responseCache.getCacheEntry(target, request);
848         } catch (final IOException ioe) {
849             // nop
850         }
851         if (existing == null) {
852             return false;
853         }
854         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
855         if (entryDateHeader == null) {
856             return false;
857         }
858         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
859         if (responseDateHeader == null) {
860             return false;
861         }
862         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
863         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
864         if (entryDate == null || responseDate == null) {
865             return false;
866         }
867         return responseDate.before(entryDate);
868     }
869 
870 }