View Javadoc

1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.net.URI;
31  import java.net.URISyntaxException;
32  import java.util.Date;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.http.Header;
41  import org.apache.http.HeaderElement;
42  import org.apache.http.HttpException;
43  import org.apache.http.HttpHost;
44  import org.apache.http.HttpMessage;
45  import org.apache.http.HttpRequest;
46  import org.apache.http.HttpResponse;
47  import org.apache.http.HttpStatus;
48  import org.apache.http.HttpVersion;
49  import org.apache.http.ProtocolException;
50  import org.apache.http.ProtocolVersion;
51  import org.apache.http.RequestLine;
52  import org.apache.http.annotation.ThreadSafe;
53  import org.apache.http.client.cache.CacheResponseStatus;
54  import org.apache.http.client.cache.HeaderConstants;
55  import org.apache.http.client.cache.HttpCacheContext;
56  import org.apache.http.client.cache.HttpCacheEntry;
57  import org.apache.http.client.cache.HttpCacheStorage;
58  import org.apache.http.client.cache.ResourceFactory;
59  import org.apache.http.client.methods.CloseableHttpResponse;
60  import org.apache.http.client.methods.HttpExecutionAware;
61  import org.apache.http.client.methods.HttpRequestWrapper;
62  import org.apache.http.client.protocol.HttpClientContext;
63  import org.apache.http.client.utils.DateUtils;
64  import org.apache.http.conn.routing.HttpRoute;
65  import org.apache.http.impl.execchain.ClientExecChain;
66  import org.apache.http.message.BasicHttpResponse;
67  import org.apache.http.protocol.HTTP;
68  import org.apache.http.protocol.HttpContext;
69  import org.apache.http.util.Args;
70  import org.apache.http.util.VersionInfo;
71  
72  /**
73   * Request executor in the request execution chain that is responsible for
74   * transparent client-side caching. The current implementation is conditionally
75   * compliant with HTTP/1.1 (meaning all the MUST and MUST NOTs are obeyed),
76   * although quite a lot, though not all, of the SHOULDs and SHOULD NOTs
77   * are obeyed too.
78   * <p/>
79   * Folks that would like to experiment with alternative storage backends
80   * should look at the {@link HttpCacheStorage} interface and the related
81   * package documentation there. You may also be interested in the provided
82   * {@link org.apache.http.impl.client.cache.ehcache.EhcacheHttpCacheStorage
83   * EhCache} and {@link
84   * org.apache.http.impl.client.cache.memcached.MemcachedHttpCacheStorage
85   * memcached} storage backends.
86   * <p/>
87   * Further responsibilities such as communication with the opposite
88   * endpoint is delegated to the next executor in the request execution
89   * chain.
90   *
91   * @since 4.3
92   */
93  @ThreadSafe // So long as the responseCache implementation is threadsafe
94  public class CachingExec implements ClientExecChain {
95  
96      private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
97  
98      private final AtomicLong cacheHits = new AtomicLong();
99      private final AtomicLong cacheMisses = new AtomicLong();
100     private final AtomicLong cacheUpdates = new AtomicLong();
101 
102     private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
103 
104     private final CacheConfig cacheConfig;
105     private final ClientExecChain backend;
106     private final HttpCache responseCache;
107     private final CacheValidityPolicy validityPolicy;
108     private final CachedHttpResponseGenerator responseGenerator;
109     private final CacheableRequestPolicy cacheableRequestPolicy;
110     private final CachedResponseSuitabilityChecker suitabilityChecker;
111     private final ConditionalRequestBuilder conditionalRequestBuilder;
112     private final ResponseProtocolCompliance responseCompliance;
113     private final RequestProtocolCompliance requestCompliance;
114     private final ResponseCachingPolicy responseCachingPolicy;
115 
116     private final AsynchronousValidator asynchRevalidator;
117 
118     private final Log log = LogFactory.getLog(getClass());
119 
120     public CachingExec(
121             final ClientExecChain backend,
122             final HttpCache cache,
123             final CacheConfig config) {
124         this(backend, cache, config, null);
125     }
126 
127     public CachingExec(
128             final ClientExecChain backend,
129             final HttpCache cache,
130             final CacheConfig config,
131             final AsynchronousValidator asynchRevalidator) {
132         super();
133         Args.notNull(backend, "HTTP backend");
134         Args.notNull(cache, "HttpCache");
135         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
136         this.backend = backend;
137         this.responseCache = cache;
138         this.validityPolicy = new CacheValidityPolicy();
139         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
140         this.cacheableRequestPolicy = new CacheableRequestPolicy();
141         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, this.cacheConfig);
142         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
143         this.responseCompliance = new ResponseProtocolCompliance();
144         this.requestCompliance = new RequestProtocolCompliance(this.cacheConfig.isWeakETagOnPutDeleteAllowed());
145         this.responseCachingPolicy = new ResponseCachingPolicy(
146                 this.cacheConfig.getMaxObjectSize(), this.cacheConfig.isSharedCache(),
147                 this.cacheConfig.isNeverCacheHTTP10ResponsesWithQuery(), this.cacheConfig.is303CachingEnabled());
148         this.asynchRevalidator = asynchRevalidator;
149     }
150 
151     public CachingExec(
152             final ClientExecChain backend,
153             final ResourceFactory resourceFactory,
154             final HttpCacheStorage storage,
155             final CacheConfig config) {
156         this(backend, new BasicHttpCache(resourceFactory, storage, config), config);
157     }
158 
159     public CachingExec(final ClientExecChain backend) {
160         this(backend, new BasicHttpCache(), CacheConfig.DEFAULT);
161     }
162 
163     CachingExec(
164             final ClientExecChain backend,
165             final HttpCache responseCache,
166             final CacheValidityPolicy validityPolicy,
167             final ResponseCachingPolicy responseCachingPolicy,
168             final CachedHttpResponseGenerator responseGenerator,
169             final CacheableRequestPolicy cacheableRequestPolicy,
170             final CachedResponseSuitabilityChecker suitabilityChecker,
171             final ConditionalRequestBuilder conditionalRequestBuilder,
172             final ResponseProtocolCompliance responseCompliance,
173             final RequestProtocolCompliance requestCompliance,
174             final CacheConfig config,
175             final AsynchronousValidator asynchRevalidator) {
176         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
177         this.backend = backend;
178         this.responseCache = responseCache;
179         this.validityPolicy = validityPolicy;
180         this.responseCachingPolicy = responseCachingPolicy;
181         this.responseGenerator = responseGenerator;
182         this.cacheableRequestPolicy = cacheableRequestPolicy;
183         this.suitabilityChecker = suitabilityChecker;
184         this.conditionalRequestBuilder = conditionalRequestBuilder;
185         this.responseCompliance = responseCompliance;
186         this.requestCompliance = requestCompliance;
187         this.asynchRevalidator = asynchRevalidator;
188     }
189 
190     /**
191      * Reports the number of times that the cache successfully responded
192      * to an {@link HttpRequest} without contacting the origin server.
193      * @return the number of cache hits
194      */
195     public long getCacheHits() {
196         return cacheHits.get();
197     }
198 
199     /**
200      * Reports the number of times that the cache contacted the origin
201      * server because it had no appropriate response cached.
202      * @return the number of cache misses
203      */
204     public long getCacheMisses() {
205         return cacheMisses.get();
206     }
207 
208     /**
209      * Reports the number of times that the cache was able to satisfy
210      * a response by revalidating an existing but stale cache entry.
211      * @return the number of cache revalidations
212      */
213     public long getCacheUpdates() {
214         return cacheUpdates.get();
215     }
216 
217     public CloseableHttpResponse execute(
218             final HttpRoute route,
219             final HttpRequestWrapper request) throws IOException, HttpException {
220         return execute(route, request, HttpClientContext.create(), null);
221     }
222 
223     public CloseableHttpResponse execute(
224             final HttpRoute route,
225             final HttpRequestWrapper request,
226             final HttpClientContext context) throws IOException, HttpException {
227         return execute(route, request, context, null);
228     }
229 
230     public CloseableHttpResponse execute(
231             final HttpRoute route,
232             final HttpRequestWrapper request,
233             final HttpClientContext context,
234             final HttpExecutionAware execAware) throws IOException, HttpException {
235 
236         final HttpHost target = context.getTargetHost();
237         final String via = generateViaHeader(request.getOriginal());
238 
239         // default response context
240         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
241 
242         if (clientRequestsOurOptions(request)) {
243             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
244             return Proxies.enhanceResponse(new OptionsHttp11Response());
245         }
246 
247         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(request, context);
248         if (fatalErrorResponse != null) {
249             return Proxies.enhanceResponse(fatalErrorResponse);
250         }
251 
252         requestCompliance.makeRequestCompliant(request);
253         request.addHeader("Via",via);
254 
255         flushEntriesInvalidatedByRequest(context.getTargetHost(), request);
256 
257         if (!cacheableRequestPolicy.isServableFromCache(request)) {
258             log.debug("Request is not servable from cache");
259             return callBackend(route, request, context, execAware);
260         }
261 
262         final HttpCacheEntry entry = satisfyFromCache(target, request);
263         if (entry == null) {
264             log.debug("Cache miss");
265             return handleCacheMiss(route, request, context, execAware);
266         } else {
267             return handleCacheHit(route, request, context, execAware, entry);
268         }
269     }
270 
271     private CloseableHttpResponse handleCacheHit(
272             final HttpRoute route,
273             final HttpRequestWrapper request,
274             final HttpClientContext context,
275             final HttpExecutionAware execAware,
276             final HttpCacheEntry entry) throws IOException, HttpException {
277         final HttpHost target = context.getTargetHost();
278         recordCacheHit(target, request);
279         CloseableHttpResponse out = null;
280         final Date now = getCurrentDate();
281         if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
282             log.debug("Cache hit");
283             out = generateCachedResponse(request, context, entry, now);
284         } else if (!mayCallBackend(request)) {
285             log.debug("Cache entry not suitable but only-if-cached requested");
286             out = generateGatewayTimeout(context);
287         } else if (!(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
288                 && !suitabilityChecker.isConditional(request))) {
289             log.debug("Revalidating cache entry");
290             return revalidateCacheEntry(route, request, context, execAware, entry, now);
291         } else {
292             log.debug("Cache entry not usable; calling backend");
293             return callBackend(route, request, context, execAware);
294         }
295         context.setAttribute(HttpClientContext.HTTP_ROUTE, route);
296         context.setAttribute(HttpClientContext.HTTP_TARGET_HOST, target);
297         context.setAttribute(HttpClientContext.HTTP_REQUEST, request);
298         context.setAttribute(HttpClientContext.HTTP_RESPONSE, out);
299         context.setAttribute(HttpClientContext.HTTP_REQ_SENT, Boolean.TRUE);
300         return out;
301     }
302 
303     private CloseableHttpResponse revalidateCacheEntry(
304             final HttpRoute route,
305             final HttpRequestWrapper request,
306             final HttpClientContext context,
307             final HttpExecutionAware execAware,
308             final HttpCacheEntry entry,
309             final Date now) throws HttpException {
310 
311         try {
312             if (asynchRevalidator != null
313                 && !staleResponseNotAllowed(request, entry, now)
314                 && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
315                 log.trace("Serving stale with asynchronous revalidation");
316                 final CloseableHttpResponse resp = generateCachedResponse(request, context, entry, now);
317                 asynchRevalidator.revalidateCacheEntry(this, route, request, context, execAware, entry);
318                 return resp;
319             }
320             return revalidateCacheEntry(route, request, context, execAware, entry);
321         } catch (final IOException ioex) {
322             return handleRevalidationFailure(request, context, entry, now);
323         }
324     }
325 
326     private CloseableHttpResponse handleCacheMiss(
327             final HttpRoute route,
328             final HttpRequestWrapper request,
329             final HttpClientContext context,
330             final HttpExecutionAware execAware) throws IOException, HttpException {
331         final HttpHost target = context.getTargetHost();
332         recordCacheMiss(target, request);
333 
334         if (!mayCallBackend(request)) {
335             return Proxies.enhanceResponse(
336                     new BasicHttpResponse(
337                             HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
338         }
339 
340         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
341         if (variants != null && variants.size() > 0) {
342             return negotiateResponseFromVariants(route, request, context,
343                     execAware, variants);
344         }
345 
346         return callBackend(route, request, context, execAware);
347     }
348 
349     private HttpCacheEntry satisfyFromCache(
350             final HttpHost target, final HttpRequestWrapper request) {
351         HttpCacheEntry entry = null;
352         try {
353             entry = responseCache.getCacheEntry(target, request);
354         } catch (final IOException ioe) {
355             log.warn("Unable to retrieve entries from cache", ioe);
356         }
357         return entry;
358     }
359 
360     private HttpResponse getFatallyNoncompliantResponse(
361             final HttpRequestWrapper request,
362             final HttpContext context) {
363         HttpResponse fatalErrorResponse = null;
364         final List<RequestProtocolError> fatalError = requestCompliance.requestIsFatallyNonCompliant(request);
365 
366         for (final RequestProtocolError error : fatalError) {
367             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
368             fatalErrorResponse = requestCompliance.getErrorForRequest(error);
369         }
370         return fatalErrorResponse;
371     }
372 
373     private Map<String, Variant> getExistingCacheVariants(
374             final HttpHost target,
375             final HttpRequestWrapper request) {
376         Map<String,Variant> variants = null;
377         try {
378             variants = responseCache.getVariantCacheEntriesWithEtags(target, request);
379         } catch (final IOException ioe) {
380             log.warn("Unable to retrieve variant entries from cache", ioe);
381         }
382         return variants;
383     }
384 
385     private void recordCacheMiss(final HttpHost target, final HttpRequestWrapper request) {
386         cacheMisses.getAndIncrement();
387         if (log.isTraceEnabled()) {
388             final RequestLine rl = request.getRequestLine();
389             log.trace("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
390         }
391     }
392 
393     private void recordCacheHit(final HttpHost target, final HttpRequestWrapper request) {
394         cacheHits.getAndIncrement();
395         if (log.isTraceEnabled()) {
396             final RequestLine rl = request.getRequestLine();
397             log.trace("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
398         }
399     }
400 
401     private void recordCacheUpdate(final HttpContext context) {
402         cacheUpdates.getAndIncrement();
403         setResponseStatus(context, CacheResponseStatus.VALIDATED);
404     }
405 
406     private void flushEntriesInvalidatedByRequest(
407             final HttpHost target,
408             final HttpRequestWrapper request) {
409         try {
410             responseCache.flushInvalidatedCacheEntriesFor(target, request);
411         } catch (final IOException ioe) {
412             log.warn("Unable to flush invalidated entries from cache", ioe);
413         }
414     }
415 
416     private CloseableHttpResponse generateCachedResponse(final HttpRequestWrapper request,
417             final HttpContext context, final HttpCacheEntry entry, final Date now) {
418         final CloseableHttpResponse cachedResponse;
419         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
420                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
421             cachedResponse = responseGenerator.generateNotModifiedResponse(entry);
422         } else {
423             cachedResponse = responseGenerator.generateResponse(entry);
424         }
425         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
426         if (validityPolicy.getStalenessSecs(entry, now) > 0L) {
427             cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\"");
428         }
429         return cachedResponse;
430     }
431 
432     private CloseableHttpResponse handleRevalidationFailure(
433             final HttpRequestWrapper request,
434             final HttpContext context,
435             final HttpCacheEntry entry,
436             final Date now) {
437         if (staleResponseNotAllowed(request, entry, now)) {
438             return generateGatewayTimeout(context);
439         } else {
440             return unvalidatedCacheHit(context, entry);
441         }
442     }
443 
444     private CloseableHttpResponse generateGatewayTimeout(
445             final HttpContext context) {
446         setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
447         return Proxies.enhanceResponse(new BasicHttpResponse(
448                 HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT,
449                 "Gateway Timeout"));
450     }
451 
452     private CloseableHttpResponse unvalidatedCacheHit(
453             final HttpContext context, final HttpCacheEntry entry) {
454         final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(entry);
455         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
456         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
457         return cachedResponse;
458     }
459 
460     private boolean staleResponseNotAllowed(
461             final HttpRequestWrapper request,
462             final HttpCacheEntry entry,
463             final Date now) {
464         return validityPolicy.mustRevalidate(entry)
465             || (cacheConfig.isSharedCache() && validityPolicy.proxyRevalidate(entry))
466             || explicitFreshnessRequest(request, entry, now);
467     }
468 
469     private boolean mayCallBackend(final HttpRequestWrapper request) {
470         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
471             for (final HeaderElement elt : h.getElements()) {
472                 if ("only-if-cached".equals(elt.getName())) {
473                     log.trace("Request marked only-if-cached");
474                     return false;
475                 }
476             }
477         }
478         return true;
479     }
480 
481     private boolean explicitFreshnessRequest(
482             final HttpRequestWrapper request,
483             final HttpCacheEntry entry,
484             final Date now) {
485         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
486             for(final HeaderElement elt : h.getElements()) {
487                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
488                     try {
489                         final int maxstale = Integer.parseInt(elt.getValue());
490                         final long age = validityPolicy.getCurrentAgeSecs(entry, now);
491                         final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry);
492                         if (age - lifetime > maxstale) {
493                             return true;
494                         }
495                     } catch (final NumberFormatException nfe) {
496                         return true;
497                     }
498                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
499                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
500                     return true;
501                 }
502             }
503         }
504         return false;
505     }
506 
507     private String generateViaHeader(final HttpMessage msg) {
508 
509         final ProtocolVersion pv = msg.getProtocolVersion();
510         final String existingEntry = viaHeaders.get(pv);
511         if (existingEntry != null) {
512             return existingEntry;
513         }
514 
515         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
516         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
517 
518         String value;
519         if ("http".equalsIgnoreCase(pv.getProtocol())) {
520             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getMajor(), pv.getMinor(),
521                     release);
522         } else {
523             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), pv.getMajor(),
524                     pv.getMinor(), release);
525         }
526         viaHeaders.put(pv, value);
527 
528         return value;
529     }
530 
531     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
532         if (context != null) {
533             context.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
534         }
535     }
536 
537     /**
538      * Reports whether this {@code CachingHttpClient} implementation
539      * supports byte-range requests as specified by the {@code Range}
540      * and {@code Content-Range} headers.
541      * @return {@code true} if byte-range requests are supported
542      */
543     public boolean supportsRangeAndContentRangeHeaders() {
544         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
545     }
546 
547     Date getCurrentDate() {
548         return new Date();
549     }
550 
551     boolean clientRequestsOurOptions(final HttpRequest request) {
552         final RequestLine line = request.getRequestLine();
553 
554         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
555             return false;
556         }
557 
558         if (!"*".equals(line.getUri())) {
559             return false;
560         }
561 
562         if (!"0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue())) {
563             return false;
564         }
565 
566         return true;
567     }
568 
569     CloseableHttpResponse callBackend(
570             final HttpRoute route,
571             final HttpRequestWrapper request,
572             final HttpClientContext context,
573             final HttpExecutionAware execAware) throws IOException, HttpException  {
574 
575         final Date requestDate = getCurrentDate();
576 
577         log.trace("Calling the backend");
578         final CloseableHttpResponse backendResponse = backend.execute(route, request, context, execAware);
579         try {
580             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
581             return handleBackendResponse(route, request, context, execAware,
582                     requestDate, getCurrentDate(), backendResponse);
583         } catch (final IOException ex) {
584             backendResponse.close();
585             throw ex;
586         } catch (final RuntimeException ex) {
587             backendResponse.close();
588             throw ex;
589         }
590     }
591 
592     private boolean revalidationResponseIsTooOld(final HttpResponse backendResponse,
593             final HttpCacheEntry cacheEntry) {
594         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
595         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
596         if (entryDateHeader != null && responseDateHeader != null) {
597             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
598             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
599             if (entryDate == null || respDate == null) {
600                 // either backend response or cached entry did not have a valid
601                 // Date header, so we can't tell if they are out of order
602                 // according to the origin clock; thus we can skip the
603                 // unconditional retry recommended in 13.2.6 of RFC 2616.
604                 return false;
605             }
606             if (respDate.before(entryDate)) {
607                 return true;
608             }
609         }
610         return false;
611     }
612 
613     CloseableHttpResponse negotiateResponseFromVariants(
614             final HttpRoute route,
615             final HttpRequestWrapper request,
616             final HttpClientContext context,
617             final HttpExecutionAware execAware,
618             final Map<String, Variant> variants) throws IOException, HttpException {
619         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder
620             .buildConditionalRequestFromVariants(request, variants);
621 
622         final Date requestDate = getCurrentDate();
623         final CloseableHttpResponse backendResponse = backend.execute(
624                 route, conditionalRequest, context, execAware);
625         try {
626             final Date responseDate = getCurrentDate();
627 
628             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
629 
630             if (backendResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
631                 return handleBackendResponse(
632                         route, request, context, execAware,
633                         requestDate, responseDate, backendResponse);
634             }
635 
636             final Header resultEtagHeader = backendResponse.getFirstHeader(HeaderConstants.ETAG);
637             if (resultEtagHeader == null) {
638                 log.warn("304 response did not contain ETag");
639                 IOUtils.consume(backendResponse.getEntity());
640                 backendResponse.close();
641                 return callBackend(route, request, context, execAware);
642             }
643 
644             final String resultEtag = resultEtagHeader.getValue();
645             final Variant matchingVariant = variants.get(resultEtag);
646             if (matchingVariant == null) {
647                 log.debug("304 response did not contain ETag matching one sent in If-None-Match");
648                 IOUtils.consume(backendResponse.getEntity());
649                 backendResponse.close();
650                 return callBackend(route, request, context, execAware);
651             }
652 
653             final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
654 
655             if (revalidationResponseIsTooOld(backendResponse, matchedEntry)) {
656                 IOUtils.consume(backendResponse.getEntity());
657                 backendResponse.close();
658                 return retryRequestUnconditionally(route, request, context, execAware, matchedEntry);
659             }
660 
661             recordCacheUpdate(context);
662 
663             final HttpCacheEntry responseEntry = getUpdatedVariantEntry(
664                 context.getTargetHost(), conditionalRequest, requestDate, responseDate,
665                     backendResponse, matchingVariant, matchedEntry);
666             backendResponse.close();
667 
668             final CloseableHttpResponse resp = responseGenerator.generateResponse(responseEntry);
669             tryToUpdateVariantMap(context.getTargetHost(), request, matchingVariant);
670 
671             if (shouldSendNotModifiedResponse(request, responseEntry)) {
672                 return responseGenerator.generateNotModifiedResponse(responseEntry);
673             }
674             return resp;
675         } catch (final IOException ex) {
676             backendResponse.close();
677             throw ex;
678         } catch (final RuntimeException ex) {
679             backendResponse.close();
680             throw ex;
681         }
682     }
683 
684     private CloseableHttpResponse retryRequestUnconditionally(
685             final HttpRoute route,
686             final HttpRequestWrapper request,
687             final HttpClientContext context,
688             final HttpExecutionAware execAware,
689             final HttpCacheEntry matchedEntry) throws IOException, HttpException {
690         final HttpRequestWrapper unconditional = conditionalRequestBuilder
691             .buildUnconditionalRequest(request, matchedEntry);
692         return callBackend(route, unconditional, context, execAware);
693     }
694 
695     private HttpCacheEntry getUpdatedVariantEntry(
696             final HttpHost target,
697             final HttpRequestWrapper conditionalRequest,
698             final Date requestDate,
699             final Date responseDate,
700             final CloseableHttpResponse backendResponse,
701             final Variant matchingVariant,
702             final HttpCacheEntry matchedEntry) throws IOException {
703         HttpCacheEntry responseEntry = matchedEntry;
704         try {
705             responseEntry = responseCache.updateVariantCacheEntry(target, conditionalRequest,
706                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
707         } catch (final IOException ioe) {
708             log.warn("Could not update cache entry", ioe);
709         } finally {
710             backendResponse.close();
711         }
712         return responseEntry;
713     }
714 
715     private void tryToUpdateVariantMap(
716             final HttpHost target,
717             final HttpRequestWrapper request,
718             final Variant matchingVariant) {
719         try {
720             responseCache.reuseVariantEntryFor(target, request, matchingVariant);
721         } catch (final IOException ioe) {
722             log.warn("Could not update cache entry to reuse variant", ioe);
723         }
724     }
725 
726     private boolean shouldSendNotModifiedResponse(
727             final HttpRequestWrapper request,
728             final HttpCacheEntry responseEntry) {
729         return (suitabilityChecker.isConditional(request)
730                 && suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
731     }
732 
733     CloseableHttpResponse revalidateCacheEntry(
734             final HttpRoute route,
735             final HttpRequestWrapper request,
736             final HttpClientContext context,
737             final HttpExecutionAware execAware,
738             final HttpCacheEntry cacheEntry) throws IOException, HttpException {
739 
740         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
741         final URI uri = conditionalRequest.getURI();
742         if (uri != null) {
743             try {
744                 conditionalRequest.setURI(InternalURIUtils.rewriteURIForRoute(uri, route));
745             } catch (final URISyntaxException ex) {
746                 throw new ProtocolException("Invalid URI: " + uri, ex);
747             }
748         }
749 
750         Date requestDate = getCurrentDate();
751         CloseableHttpResponse backendResponse = backend.execute(
752                 route, conditionalRequest, context, execAware);
753         Date responseDate = getCurrentDate();
754 
755         if (revalidationResponseIsTooOld(backendResponse, cacheEntry)) {
756             backendResponse.close();
757             final HttpRequestWrapper unconditional = conditionalRequestBuilder
758                 .buildUnconditionalRequest(request, cacheEntry);
759             requestDate = getCurrentDate();
760             backendResponse = backend.execute(route, unconditional, context, execAware);
761             responseDate = getCurrentDate();
762         }
763 
764         backendResponse.addHeader(HeaderConstants.VIA, generateViaHeader(backendResponse));
765 
766         final int statusCode = backendResponse.getStatusLine().getStatusCode();
767         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
768             recordCacheUpdate(context);
769         }
770 
771         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
772             final HttpCacheEntry updatedEntry = responseCache.updateCacheEntry(
773                     context.getTargetHost(), request, cacheEntry,
774                     backendResponse, requestDate, responseDate);
775             if (suitabilityChecker.isConditional(request)
776                     && suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
777                 return responseGenerator
778                         .generateNotModifiedResponse(updatedEntry);
779             }
780             return responseGenerator.generateResponse(updatedEntry);
781         }
782 
783         if (staleIfErrorAppliesTo(statusCode)
784             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
785             && validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
786             try {
787                 final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(cacheEntry);
788                 cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
789                 return cachedResponse;
790             } finally {
791                 backendResponse.close();
792             }
793         }
794         return handleBackendResponse(
795                 route, conditionalRequest, context, execAware,
796                 requestDate, responseDate, backendResponse);
797     }
798 
799     private boolean staleIfErrorAppliesTo(final int statusCode) {
800         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
801                 || statusCode == HttpStatus.SC_BAD_GATEWAY
802                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
803                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
804     }
805 
806     CloseableHttpResponse handleBackendResponse(
807             final HttpRoute route,
808             final HttpRequestWrapper request,
809             final HttpClientContext context,
810             final HttpExecutionAware execAware,
811             final Date requestDate,
812             final Date responseDate,
813             final CloseableHttpResponse backendResponse) throws IOException {
814 
815         log.trace("Handling Backend response");
816         responseCompliance.ensureProtocolCompliance(request, backendResponse);
817 
818         final HttpHost target = context.getTargetHost();
819         final boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse);
820         responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
821         if (cacheable && !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
822             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
823             return responseCache.cacheAndReturnResponse(target, request,
824                     backendResponse, requestDate, responseDate);
825         }
826         if (!cacheable) {
827             try {
828                 responseCache.flushCacheEntriesFor(target, request);
829             } catch (final IOException ioe) {
830                 log.warn("Unable to flush invalid cache entries", ioe);
831             }
832         }
833         return backendResponse;
834     }
835 
836     /**
837      * For 304 Not modified responses, adds a "Last-Modified" header with the
838      * value of the "If-Modified-Since" header passed in the request. This
839      * header is required to be able to reuse match the cache entry for
840      * subsequent requests but as defined in http specifications it is not
841      * included in 304 responses by backend servers. This header will not be
842      * included in the resulting response.
843      */
844     private void storeRequestIfModifiedSinceFor304Response(
845             final HttpRequest request, final HttpResponse backendResponse) {
846         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
847             final Header h = request.getFirstHeader("If-Modified-Since");
848             if (h != null) {
849                 backendResponse.addHeader("Last-Modified", h.getValue());
850             }
851         }
852     }
853 
854     private boolean alreadyHaveNewerCacheEntry(final HttpHost target, final HttpRequestWrapper request,
855             final HttpResponse backendResponse) {
856         HttpCacheEntry existing = null;
857         try {
858             existing = responseCache.getCacheEntry(target, request);
859         } catch (final IOException ioe) {
860             // nop
861         }
862         if (existing == null) {
863             return false;
864         }
865         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
866         if (entryDateHeader == null) {
867             return false;
868         }
869         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
870         if (responseDateHeader == null) {
871             return false;
872         }
873         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
874         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
875         if (entryDate == null || responseDate == null) {
876             return false;
877         }
878         return responseDate.before(entryDate);
879     }
880 
881 }