View Javadoc

1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.net.URI;
31  import java.net.URISyntaxException;
32  import java.util.Date;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.http.Header;
41  import org.apache.http.HeaderElement;
42  import org.apache.http.HttpException;
43  import org.apache.http.HttpHost;
44  import org.apache.http.HttpMessage;
45  import org.apache.http.HttpRequest;
46  import org.apache.http.HttpResponse;
47  import org.apache.http.HttpStatus;
48  import org.apache.http.HttpVersion;
49  import org.apache.http.ProtocolException;
50  import org.apache.http.ProtocolVersion;
51  import org.apache.http.RequestLine;
52  import org.apache.http.annotation.ThreadSafe;
53  import org.apache.http.client.cache.CacheResponseStatus;
54  import org.apache.http.client.cache.HeaderConstants;
55  import org.apache.http.client.cache.HttpCacheContext;
56  import org.apache.http.client.cache.HttpCacheEntry;
57  import org.apache.http.client.cache.HttpCacheStorage;
58  import org.apache.http.client.cache.ResourceFactory;
59  import org.apache.http.client.methods.CloseableHttpResponse;
60  import org.apache.http.client.methods.HttpExecutionAware;
61  import org.apache.http.client.methods.HttpRequestWrapper;
62  import org.apache.http.client.protocol.HttpClientContext;
63  import org.apache.http.client.utils.DateUtils;
64  import org.apache.http.client.utils.URIUtils;
65  import org.apache.http.conn.routing.HttpRoute;
66  import org.apache.http.impl.execchain.ClientExecChain;
67  import org.apache.http.message.BasicHttpResponse;
68  import org.apache.http.protocol.HTTP;
69  import org.apache.http.protocol.HttpContext;
70  import org.apache.http.protocol.HttpCoreContext;
71  import org.apache.http.util.Args;
72  import org.apache.http.util.VersionInfo;
73  
74  /**
75   * <p>
76   * Request executor in the request execution chain that is responsible for
77   * transparent client-side caching.
78   * </p>
79   * <p>
80   * The current implementation is conditionally
81   * compliant with HTTP/1.1 (meaning all the MUST and MUST NOTs are obeyed),
82   * although quite a lot, though not all, of the SHOULDs and SHOULD NOTs
83   * are obeyed too.
84   * </p>
85   * <p>
86   * Folks that would like to experiment with alternative storage backends
87   * should look at the {@link HttpCacheStorage} interface and the related
88   * package documentation there. You may also be interested in the provided
89   * {@link org.apache.http.impl.client.cache.ehcache.EhcacheHttpCacheStorage
90   * EhCache} and {@link
91   * org.apache.http.impl.client.cache.memcached.MemcachedHttpCacheStorage
92   * memcached} storage backends.
93   * </p>
94   * <p>
95   * Further responsibilities such as communication with the opposite
96   * endpoint is delegated to the next executor in the request execution
97   * chain.
98   * </p>
99   *
100  * @since 4.3
101  */
102 @ThreadSafe // So long as the responseCache implementation is threadsafe
103 public class CachingExec implements ClientExecChain {
104 
105     private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
106 
107     private final AtomicLong cacheHits = new AtomicLong();
108     private final AtomicLong cacheMisses = new AtomicLong();
109     private final AtomicLong cacheUpdates = new AtomicLong();
110 
111     private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
112 
113     private final CacheConfig cacheConfig;
114     private final ClientExecChain backend;
115     private final HttpCache responseCache;
116     private final CacheValidityPolicy validityPolicy;
117     private final CachedHttpResponseGenerator responseGenerator;
118     private final CacheableRequestPolicy cacheableRequestPolicy;
119     private final CachedResponseSuitabilityChecker suitabilityChecker;
120     private final ConditionalRequestBuilder conditionalRequestBuilder;
121     private final ResponseProtocolCompliance responseCompliance;
122     private final RequestProtocolCompliance requestCompliance;
123     private final ResponseCachingPolicy responseCachingPolicy;
124 
125     private final AsynchronousValidator asynchRevalidator;
126 
127     private final Log log = LogFactory.getLog(getClass());
128 
129     public CachingExec(
130             final ClientExecChain backend,
131             final HttpCache cache,
132             final CacheConfig config) {
133         this(backend, cache, config, null);
134     }
135 
136     public CachingExec(
137             final ClientExecChain backend,
138             final HttpCache cache,
139             final CacheConfig config,
140             final AsynchronousValidator asynchRevalidator) {
141         super();
142         Args.notNull(backend, "HTTP backend");
143         Args.notNull(cache, "HttpCache");
144         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
145         this.backend = backend;
146         this.responseCache = cache;
147         this.validityPolicy = new CacheValidityPolicy();
148         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
149         this.cacheableRequestPolicy = new CacheableRequestPolicy();
150         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, this.cacheConfig);
151         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
152         this.responseCompliance = new ResponseProtocolCompliance();
153         this.requestCompliance = new RequestProtocolCompliance(this.cacheConfig.isWeakETagOnPutDeleteAllowed());
154         this.responseCachingPolicy = new ResponseCachingPolicy(
155                 this.cacheConfig.getMaxObjectSize(), this.cacheConfig.isSharedCache(),
156                 this.cacheConfig.isNeverCacheHTTP10ResponsesWithQuery(), this.cacheConfig.is303CachingEnabled());
157         this.asynchRevalidator = asynchRevalidator;
158     }
159 
160     public CachingExec(
161             final ClientExecChain backend,
162             final ResourceFactory resourceFactory,
163             final HttpCacheStorage storage,
164             final CacheConfig config) {
165         this(backend, new BasicHttpCache(resourceFactory, storage, config), config);
166     }
167 
168     public CachingExec(final ClientExecChain backend) {
169         this(backend, new BasicHttpCache(), CacheConfig.DEFAULT);
170     }
171 
172     CachingExec(
173             final ClientExecChain backend,
174             final HttpCache responseCache,
175             final CacheValidityPolicy validityPolicy,
176             final ResponseCachingPolicy responseCachingPolicy,
177             final CachedHttpResponseGenerator responseGenerator,
178             final CacheableRequestPolicy cacheableRequestPolicy,
179             final CachedResponseSuitabilityChecker suitabilityChecker,
180             final ConditionalRequestBuilder conditionalRequestBuilder,
181             final ResponseProtocolCompliance responseCompliance,
182             final RequestProtocolCompliance requestCompliance,
183             final CacheConfig config,
184             final AsynchronousValidator asynchRevalidator) {
185         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
186         this.backend = backend;
187         this.responseCache = responseCache;
188         this.validityPolicy = validityPolicy;
189         this.responseCachingPolicy = responseCachingPolicy;
190         this.responseGenerator = responseGenerator;
191         this.cacheableRequestPolicy = cacheableRequestPolicy;
192         this.suitabilityChecker = suitabilityChecker;
193         this.conditionalRequestBuilder = conditionalRequestBuilder;
194         this.responseCompliance = responseCompliance;
195         this.requestCompliance = requestCompliance;
196         this.asynchRevalidator = asynchRevalidator;
197     }
198 
199     /**
200      * Reports the number of times that the cache successfully responded
201      * to an {@link HttpRequest} without contacting the origin server.
202      * @return the number of cache hits
203      */
204     public long getCacheHits() {
205         return cacheHits.get();
206     }
207 
208     /**
209      * Reports the number of times that the cache contacted the origin
210      * server because it had no appropriate response cached.
211      * @return the number of cache misses
212      */
213     public long getCacheMisses() {
214         return cacheMisses.get();
215     }
216 
217     /**
218      * Reports the number of times that the cache was able to satisfy
219      * a response by revalidating an existing but stale cache entry.
220      * @return the number of cache revalidations
221      */
222     public long getCacheUpdates() {
223         return cacheUpdates.get();
224     }
225 
226     public CloseableHttpResponse execute(
227             final HttpRoute route,
228             final HttpRequestWrapper request) throws IOException, HttpException {
229         return execute(route, request, HttpClientContext.create(), null);
230     }
231 
232     public CloseableHttpResponse execute(
233             final HttpRoute route,
234             final HttpRequestWrapper request,
235             final HttpClientContext context) throws IOException, HttpException {
236         return execute(route, request, context, null);
237     }
238 
239     @Override
240     public CloseableHttpResponse execute(
241             final HttpRoute route,
242             final HttpRequestWrapper request,
243             final HttpClientContext context,
244             final HttpExecutionAware execAware) throws IOException, HttpException {
245 
246         final HttpHost target = context.getTargetHost();
247         final String via = generateViaHeader(request.getOriginal());
248 
249         // default response context
250         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
251 
252         if (clientRequestsOurOptions(request)) {
253             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
254             return Proxies.enhanceResponse(new OptionsHttp11Response());
255         }
256 
257         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(request, context);
258         if (fatalErrorResponse != null) {
259             return Proxies.enhanceResponse(fatalErrorResponse);
260         }
261 
262         requestCompliance.makeRequestCompliant(request);
263         request.addHeader("Via",via);
264 
265         flushEntriesInvalidatedByRequest(context.getTargetHost(), request);
266 
267         if (!cacheableRequestPolicy.isServableFromCache(request)) {
268             log.debug("Request is not servable from cache");
269             return callBackend(route, request, context, execAware);
270         }
271 
272         final HttpCacheEntry entry = satisfyFromCache(target, request);
273         if (entry == null) {
274             log.debug("Cache miss");
275             return handleCacheMiss(route, request, context, execAware);
276         } else {
277             return handleCacheHit(route, request, context, execAware, entry);
278         }
279     }
280 
281     private CloseableHttpResponse handleCacheHit(
282             final HttpRoute route,
283             final HttpRequestWrapper request,
284             final HttpClientContext context,
285             final HttpExecutionAware execAware,
286             final HttpCacheEntry entry) throws IOException, HttpException {
287         final HttpHost target = context.getTargetHost();
288         recordCacheHit(target, request);
289         CloseableHttpResponse out = null;
290         final Date now = getCurrentDate();
291         if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
292             log.debug("Cache hit");
293             out = generateCachedResponse(request, context, entry, now);
294         } else if (!mayCallBackend(request)) {
295             log.debug("Cache entry not suitable but only-if-cached requested");
296             out = generateGatewayTimeout(context);
297         } else if (!(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
298                 && !suitabilityChecker.isConditional(request))) {
299             log.debug("Revalidating cache entry");
300             return revalidateCacheEntry(route, request, context, execAware, entry, now);
301         } else {
302             log.debug("Cache entry not usable; calling backend");
303             return callBackend(route, request, context, execAware);
304         }
305         context.setAttribute(HttpClientContext.HTTP_ROUTE, route);
306         context.setAttribute(HttpCoreContext.HTTP_TARGET_HOST, target);
307         context.setAttribute(HttpCoreContext.HTTP_REQUEST, request);
308         context.setAttribute(HttpCoreContext.HTTP_RESPONSE, out);
309         context.setAttribute(HttpCoreContext.HTTP_REQ_SENT, Boolean.TRUE);
310         return out;
311     }
312 
313     private CloseableHttpResponse revalidateCacheEntry(
314             final HttpRoute route,
315             final HttpRequestWrapper request,
316             final HttpClientContext context,
317             final HttpExecutionAware execAware,
318             final HttpCacheEntry entry,
319             final Date now) throws HttpException {
320 
321         try {
322             if (asynchRevalidator != null
323                 && !staleResponseNotAllowed(request, entry, now)
324                 && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
325                 log.trace("Serving stale with asynchronous revalidation");
326                 final CloseableHttpResponse resp = generateCachedResponse(request, context, entry, now);
327                 asynchRevalidator.revalidateCacheEntry(this, route, request, context, execAware, entry);
328                 return resp;
329             }
330             return revalidateCacheEntry(route, request, context, execAware, entry);
331         } catch (final IOException ioex) {
332             return handleRevalidationFailure(request, context, entry, now);
333         }
334     }
335 
336     private CloseableHttpResponse handleCacheMiss(
337             final HttpRoute route,
338             final HttpRequestWrapper request,
339             final HttpClientContext context,
340             final HttpExecutionAware execAware) throws IOException, HttpException {
341         final HttpHost target = context.getTargetHost();
342         recordCacheMiss(target, request);
343 
344         if (!mayCallBackend(request)) {
345             return Proxies.enhanceResponse(
346                     new BasicHttpResponse(
347                             HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
348         }
349 
350         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
351         if (variants != null && !variants.isEmpty()) {
352             return negotiateResponseFromVariants(route, request, context,
353                     execAware, variants);
354         }
355 
356         return callBackend(route, request, context, execAware);
357     }
358 
359     private HttpCacheEntry satisfyFromCache(
360             final HttpHost target, final HttpRequestWrapper request) {
361         HttpCacheEntry entry = null;
362         try {
363             entry = responseCache.getCacheEntry(target, request);
364         } catch (final IOException ioe) {
365             log.warn("Unable to retrieve entries from cache", ioe);
366         }
367         return entry;
368     }
369 
370     private HttpResponse getFatallyNoncompliantResponse(
371             final HttpRequestWrapper request,
372             final HttpContext context) {
373         HttpResponse fatalErrorResponse = null;
374         final List<RequestProtocolError> fatalError = requestCompliance.requestIsFatallyNonCompliant(request);
375 
376         for (final RequestProtocolError error : fatalError) {
377             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
378             fatalErrorResponse = requestCompliance.getErrorForRequest(error);
379         }
380         return fatalErrorResponse;
381     }
382 
383     private Map<String, Variant> getExistingCacheVariants(
384             final HttpHost target,
385             final HttpRequestWrapper request) {
386         Map<String,Variant> variants = null;
387         try {
388             variants = responseCache.getVariantCacheEntriesWithEtags(target, request);
389         } catch (final IOException ioe) {
390             log.warn("Unable to retrieve variant entries from cache", ioe);
391         }
392         return variants;
393     }
394 
395     private void recordCacheMiss(final HttpHost target, final HttpRequestWrapper request) {
396         cacheMisses.getAndIncrement();
397         if (log.isTraceEnabled()) {
398             final RequestLine rl = request.getRequestLine();
399             log.trace("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
400         }
401     }
402 
403     private void recordCacheHit(final HttpHost target, final HttpRequestWrapper request) {
404         cacheHits.getAndIncrement();
405         if (log.isTraceEnabled()) {
406             final RequestLine rl = request.getRequestLine();
407             log.trace("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
408         }
409     }
410 
411     private void recordCacheUpdate(final HttpContext context) {
412         cacheUpdates.getAndIncrement();
413         setResponseStatus(context, CacheResponseStatus.VALIDATED);
414     }
415 
416     private void flushEntriesInvalidatedByRequest(
417             final HttpHost target,
418             final HttpRequestWrapper request) {
419         try {
420             responseCache.flushInvalidatedCacheEntriesFor(target, request);
421         } catch (final IOException ioe) {
422             log.warn("Unable to flush invalidated entries from cache", ioe);
423         }
424     }
425 
426     private CloseableHttpResponse generateCachedResponse(final HttpRequestWrapper request,
427             final HttpContext context, final HttpCacheEntry entry, final Date now) {
428         final CloseableHttpResponse cachedResponse;
429         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
430                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
431             cachedResponse = responseGenerator.generateNotModifiedResponse(entry);
432         } else {
433             cachedResponse = responseGenerator.generateResponse(request, entry);
434         }
435         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
436         if (validityPolicy.getStalenessSecs(entry, now) > 0L) {
437             cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\"");
438         }
439         return cachedResponse;
440     }
441 
442     private CloseableHttpResponse handleRevalidationFailure(
443             final HttpRequestWrapper request,
444             final HttpContext context,
445             final HttpCacheEntry entry,
446             final Date now) {
447         if (staleResponseNotAllowed(request, entry, now)) {
448             return generateGatewayTimeout(context);
449         } else {
450             return unvalidatedCacheHit(request, context, entry);
451         }
452     }
453 
454     private CloseableHttpResponse generateGatewayTimeout(
455             final HttpContext context) {
456         setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
457         return Proxies.enhanceResponse(new BasicHttpResponse(
458                 HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT,
459                 "Gateway Timeout"));
460     }
461 
462     private CloseableHttpResponse unvalidatedCacheHit(
463             final HttpRequestWrapper request,
464             final HttpContext context,
465             final HttpCacheEntry entry) {
466         final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(request, entry);
467         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
468         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
469         return cachedResponse;
470     }
471 
472     private boolean staleResponseNotAllowed(
473             final HttpRequestWrapper request,
474             final HttpCacheEntry entry,
475             final Date now) {
476         return validityPolicy.mustRevalidate(entry)
477             || (cacheConfig.isSharedCache() && validityPolicy.proxyRevalidate(entry))
478             || explicitFreshnessRequest(request, entry, now);
479     }
480 
481     private boolean mayCallBackend(final HttpRequestWrapper request) {
482         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
483             for (final HeaderElement elt : h.getElements()) {
484                 if ("only-if-cached".equals(elt.getName())) {
485                     log.trace("Request marked only-if-cached");
486                     return false;
487                 }
488             }
489         }
490         return true;
491     }
492 
493     private boolean explicitFreshnessRequest(
494             final HttpRequestWrapper request,
495             final HttpCacheEntry entry,
496             final Date now) {
497         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
498             for(final HeaderElement elt : h.getElements()) {
499                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
500                     try {
501                         final int maxstale = Integer.parseInt(elt.getValue());
502                         final long age = validityPolicy.getCurrentAgeSecs(entry, now);
503                         final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry);
504                         if (age - lifetime > maxstale) {
505                             return true;
506                         }
507                     } catch (final NumberFormatException nfe) {
508                         return true;
509                     }
510                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
511                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
512                     return true;
513                 }
514             }
515         }
516         return false;
517     }
518 
519     private String generateViaHeader(final HttpMessage msg) {
520 
521         final ProtocolVersion pv = msg.getProtocolVersion();
522         final String existingEntry = viaHeaders.get(pv);
523         if (existingEntry != null) {
524             return existingEntry;
525         }
526 
527         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
528         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
529 
530         String value;
531         final int major = pv.getMajor();
532         final int minor = pv.getMinor();
533         if ("http".equalsIgnoreCase(pv.getProtocol())) {
534             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", major, minor,
535                     release);
536         } else {
537             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), major,
538                     minor, release);
539         }
540         viaHeaders.put(pv, value);
541 
542         return value;
543     }
544 
545     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
546         if (context != null) {
547             context.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
548         }
549     }
550 
551     /**
552      * Reports whether this {@code CachingHttpClient} implementation
553      * supports byte-range requests as specified by the {@code Range}
554      * and {@code Content-Range} headers.
555      * @return {@code true} if byte-range requests are supported
556      */
557     public boolean supportsRangeAndContentRangeHeaders() {
558         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
559     }
560 
561     Date getCurrentDate() {
562         return new Date();
563     }
564 
565     boolean clientRequestsOurOptions(final HttpRequest request) {
566         final RequestLine line = request.getRequestLine();
567 
568         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
569             return false;
570         }
571 
572         if (!"*".equals(line.getUri())) {
573             return false;
574         }
575 
576         if (!"0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue())) {
577             return false;
578         }
579 
580         return true;
581     }
582 
583     CloseableHttpResponse callBackend(
584             final HttpRoute route,
585             final HttpRequestWrapper request,
586             final HttpClientContext context,
587             final HttpExecutionAware execAware) throws IOException, HttpException  {
588 
589         final Date requestDate = getCurrentDate();
590 
591         log.trace("Calling the backend");
592         final CloseableHttpResponse backendResponse = backend.execute(route, request, context, execAware);
593         try {
594             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
595             return handleBackendResponse(request, context, requestDate, getCurrentDate(),
596                     backendResponse);
597         } catch (final IOException ex) {
598             backendResponse.close();
599             throw ex;
600         } catch (final RuntimeException ex) {
601             backendResponse.close();
602             throw ex;
603         }
604     }
605 
606     private boolean revalidationResponseIsTooOld(final HttpResponse backendResponse,
607             final HttpCacheEntry cacheEntry) {
608         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
609         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
610         if (entryDateHeader != null && responseDateHeader != null) {
611             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
612             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
613             if (entryDate == null || respDate == null) {
614                 // either backend response or cached entry did not have a valid
615                 // Date header, so we can't tell if they are out of order
616                 // according to the origin clock; thus we can skip the
617                 // unconditional retry recommended in 13.2.6 of RFC 2616.
618                 return false;
619             }
620             if (respDate.before(entryDate)) {
621                 return true;
622             }
623         }
624         return false;
625     }
626 
627     CloseableHttpResponse negotiateResponseFromVariants(
628             final HttpRoute route,
629             final HttpRequestWrapper request,
630             final HttpClientContext context,
631             final HttpExecutionAware execAware,
632             final Map<String, Variant> variants) throws IOException, HttpException {
633         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder
634             .buildConditionalRequestFromVariants(request, variants);
635 
636         final Date requestDate = getCurrentDate();
637         final CloseableHttpResponse backendResponse = backend.execute(
638                 route, conditionalRequest, context, execAware);
639         try {
640             final Date responseDate = getCurrentDate();
641 
642             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
643 
644             if (backendResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
645                 return handleBackendResponse(request, context, requestDate, responseDate,
646                         backendResponse);
647             }
648 
649             final Header resultEtagHeader = backendResponse.getFirstHeader(HeaderConstants.ETAG);
650             if (resultEtagHeader == null) {
651                 log.warn("304 response did not contain ETag");
652                 IOUtils.consume(backendResponse.getEntity());
653                 backendResponse.close();
654                 return callBackend(route, request, context, execAware);
655             }
656 
657             final String resultEtag = resultEtagHeader.getValue();
658             final Variant matchingVariant = variants.get(resultEtag);
659             if (matchingVariant == null) {
660                 log.debug("304 response did not contain ETag matching one sent in If-None-Match");
661                 IOUtils.consume(backendResponse.getEntity());
662                 backendResponse.close();
663                 return callBackend(route, request, context, execAware);
664             }
665 
666             final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
667 
668             if (revalidationResponseIsTooOld(backendResponse, matchedEntry)) {
669                 IOUtils.consume(backendResponse.getEntity());
670                 backendResponse.close();
671                 return retryRequestUnconditionally(route, request, context, execAware, matchedEntry);
672             }
673 
674             recordCacheUpdate(context);
675 
676             final HttpCacheEntry responseEntry = getUpdatedVariantEntry(
677                 context.getTargetHost(), conditionalRequest, requestDate, responseDate,
678                     backendResponse, matchingVariant, matchedEntry);
679             backendResponse.close();
680 
681             final CloseableHttpResponse resp = responseGenerator.generateResponse(request, responseEntry);
682             tryToUpdateVariantMap(context.getTargetHost(), request, matchingVariant);
683 
684             if (shouldSendNotModifiedResponse(request, responseEntry)) {
685                 return responseGenerator.generateNotModifiedResponse(responseEntry);
686             }
687             return resp;
688         } catch (final IOException ex) {
689             backendResponse.close();
690             throw ex;
691         } catch (final RuntimeException ex) {
692             backendResponse.close();
693             throw ex;
694         }
695     }
696 
697     private CloseableHttpResponse retryRequestUnconditionally(
698             final HttpRoute route,
699             final HttpRequestWrapper request,
700             final HttpClientContext context,
701             final HttpExecutionAware execAware,
702             final HttpCacheEntry matchedEntry) throws IOException, HttpException {
703         final HttpRequestWrapper unconditional = conditionalRequestBuilder
704             .buildUnconditionalRequest(request, matchedEntry);
705         return callBackend(route, unconditional, context, execAware);
706     }
707 
708     private HttpCacheEntry getUpdatedVariantEntry(
709             final HttpHost target,
710             final HttpRequestWrapper conditionalRequest,
711             final Date requestDate,
712             final Date responseDate,
713             final CloseableHttpResponse backendResponse,
714             final Variant matchingVariant,
715             final HttpCacheEntry matchedEntry) throws IOException {
716         HttpCacheEntry responseEntry = matchedEntry;
717         try {
718             responseEntry = responseCache.updateVariantCacheEntry(target, conditionalRequest,
719                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
720         } catch (final IOException ioe) {
721             log.warn("Could not update cache entry", ioe);
722         } finally {
723             backendResponse.close();
724         }
725         return responseEntry;
726     }
727 
728     private void tryToUpdateVariantMap(
729             final HttpHost target,
730             final HttpRequestWrapper request,
731             final Variant matchingVariant) {
732         try {
733             responseCache.reuseVariantEntryFor(target, request, matchingVariant);
734         } catch (final IOException ioe) {
735             log.warn("Could not update cache entry to reuse variant", ioe);
736         }
737     }
738 
739     private boolean shouldSendNotModifiedResponse(
740             final HttpRequestWrapper request,
741             final HttpCacheEntry responseEntry) {
742         return (suitabilityChecker.isConditional(request)
743                 && suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
744     }
745 
746     CloseableHttpResponse revalidateCacheEntry(
747             final HttpRoute route,
748             final HttpRequestWrapper request,
749             final HttpClientContext context,
750             final HttpExecutionAware execAware,
751             final HttpCacheEntry cacheEntry) throws IOException, HttpException {
752 
753         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
754         final URI uri = conditionalRequest.getURI();
755         if (uri != null) {
756             try {
757                 conditionalRequest.setURI(URIUtils.rewriteURIForRoute(uri, route));
758             } catch (final URISyntaxException ex) {
759                 throw new ProtocolException("Invalid URI: " + uri, ex);
760             }
761         }
762 
763         Date requestDate = getCurrentDate();
764         CloseableHttpResponse backendResponse = backend.execute(
765                 route, conditionalRequest, context, execAware);
766         Date responseDate = getCurrentDate();
767 
768         if (revalidationResponseIsTooOld(backendResponse, cacheEntry)) {
769             backendResponse.close();
770             final HttpRequestWrapper unconditional = conditionalRequestBuilder
771                 .buildUnconditionalRequest(request, cacheEntry);
772             requestDate = getCurrentDate();
773             backendResponse = backend.execute(route, unconditional, context, execAware);
774             responseDate = getCurrentDate();
775         }
776 
777         backendResponse.addHeader(HeaderConstants.VIA, generateViaHeader(backendResponse));
778 
779         final int statusCode = backendResponse.getStatusLine().getStatusCode();
780         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
781             recordCacheUpdate(context);
782         }
783 
784         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
785             final HttpCacheEntry updatedEntry = responseCache.updateCacheEntry(
786                     context.getTargetHost(), request, cacheEntry,
787                     backendResponse, requestDate, responseDate);
788             if (suitabilityChecker.isConditional(request)
789                     && suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
790                 return responseGenerator
791                         .generateNotModifiedResponse(updatedEntry);
792             }
793             return responseGenerator.generateResponse(request, updatedEntry);
794         }
795 
796         if (staleIfErrorAppliesTo(statusCode)
797             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
798             && validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
799             try {
800                 final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(request, cacheEntry);
801                 cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
802                 return cachedResponse;
803             } finally {
804                 backendResponse.close();
805             }
806         }
807         return handleBackendResponse(conditionalRequest, context, requestDate, responseDate,
808                 backendResponse);
809     }
810 
811     private boolean staleIfErrorAppliesTo(final int statusCode) {
812         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
813                 || statusCode == HttpStatus.SC_BAD_GATEWAY
814                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
815                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
816     }
817 
818     CloseableHttpResponse handleBackendResponse(
819             final HttpRequestWrapper request,
820             final HttpClientContext context,
821             final Date requestDate,
822             final Date responseDate,
823             final CloseableHttpResponse backendResponse) throws IOException {
824 
825         log.trace("Handling Backend response");
826         responseCompliance.ensureProtocolCompliance(request, backendResponse);
827 
828         final HttpHost target = context.getTargetHost();
829         final boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse);
830         responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
831         if (cacheable && !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
832             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
833             return responseCache.cacheAndReturnResponse(target, request,
834                     backendResponse, requestDate, responseDate);
835         }
836         if (!cacheable) {
837             try {
838                 responseCache.flushCacheEntriesFor(target, request);
839             } catch (final IOException ioe) {
840                 log.warn("Unable to flush invalid cache entries", ioe);
841             }
842         }
843         return backendResponse;
844     }
845 
846     /**
847      * For 304 Not modified responses, adds a "Last-Modified" header with the
848      * value of the "If-Modified-Since" header passed in the request. This
849      * header is required to be able to reuse match the cache entry for
850      * subsequent requests but as defined in http specifications it is not
851      * included in 304 responses by backend servers. This header will not be
852      * included in the resulting response.
853      */
854     private void storeRequestIfModifiedSinceFor304Response(
855             final HttpRequest request, final HttpResponse backendResponse) {
856         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
857             final Header h = request.getFirstHeader("If-Modified-Since");
858             if (h != null) {
859                 backendResponse.addHeader("Last-Modified", h.getValue());
860             }
861         }
862     }
863 
864     private boolean alreadyHaveNewerCacheEntry(final HttpHost target, final HttpRequestWrapper request,
865             final HttpResponse backendResponse) {
866         HttpCacheEntry existing = null;
867         try {
868             existing = responseCache.getCacheEntry(target, request);
869         } catch (final IOException ioe) {
870             // nop
871         }
872         if (existing == null) {
873             return false;
874         }
875         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
876         if (entryDateHeader == null) {
877             return false;
878         }
879         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
880         if (responseDateHeader == null) {
881             return false;
882         }
883         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
884         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
885         if (entryDate == null || responseDate == null) {
886             return false;
887         }
888         return responseDate.before(entryDate);
889     }
890 
891 }