View Javadoc

1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.net.URI;
31  import java.net.URISyntaxException;
32  import java.util.Date;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.http.Header;
41  import org.apache.http.HeaderElement;
42  import org.apache.http.HttpException;
43  import org.apache.http.HttpHost;
44  import org.apache.http.HttpMessage;
45  import org.apache.http.HttpRequest;
46  import org.apache.http.HttpResponse;
47  import org.apache.http.HttpStatus;
48  import org.apache.http.HttpVersion;
49  import org.apache.http.ProtocolException;
50  import org.apache.http.ProtocolVersion;
51  import org.apache.http.RequestLine;
52  import org.apache.http.annotation.ThreadSafe;
53  import org.apache.http.client.cache.CacheResponseStatus;
54  import org.apache.http.client.cache.HeaderConstants;
55  import org.apache.http.client.cache.HttpCacheContext;
56  import org.apache.http.client.cache.HttpCacheEntry;
57  import org.apache.http.client.cache.HttpCacheStorage;
58  import org.apache.http.client.cache.ResourceFactory;
59  import org.apache.http.client.methods.CloseableHttpResponse;
60  import org.apache.http.client.methods.HttpExecutionAware;
61  import org.apache.http.client.methods.HttpRequestWrapper;
62  import org.apache.http.client.protocol.HttpClientContext;
63  import org.apache.http.client.utils.DateUtils;
64  import org.apache.http.client.utils.URIUtils;
65  import org.apache.http.conn.routing.HttpRoute;
66  import org.apache.http.impl.execchain.ClientExecChain;
67  import org.apache.http.message.BasicHttpResponse;
68  import org.apache.http.protocol.HTTP;
69  import org.apache.http.protocol.HttpContext;
70  import org.apache.http.protocol.HttpCoreContext;
71  import org.apache.http.util.Args;
72  import org.apache.http.util.VersionInfo;
73  
74  /**
75   * <p>
76   * Request executor in the request execution chain that is responsible for
77   * transparent client-side caching.
78   * </p>
79   * <p>
80   * The current implementation is conditionally
81   * compliant with HTTP/1.1 (meaning all the MUST and MUST NOTs are obeyed),
82   * although quite a lot, though not all, of the SHOULDs and SHOULD NOTs
83   * are obeyed too.
84   * </p>
85   * <p>
86   * Folks that would like to experiment with alternative storage backends
87   * should look at the {@link HttpCacheStorage} interface and the related
88   * package documentation there. You may also be interested in the provided
89   * {@link org.apache.http.impl.client.cache.ehcache.EhcacheHttpCacheStorage
90   * EhCache} and {@link
91   * org.apache.http.impl.client.cache.memcached.MemcachedHttpCacheStorage
92   * memcached} storage backends.
93   * </p>
94   * <p>
95   * Further responsibilities such as communication with the opposite
96   * endpoint is delegated to the next executor in the request execution
97   * chain.
98   * </p>
99   *
100  * @since 4.3
101  */
102 @ThreadSafe // So long as the responseCache implementation is threadsafe
103 public class CachingExec implements ClientExecChain {
104 
105     private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
106 
107     private final AtomicLong cacheHits = new AtomicLong();
108     private final AtomicLong cacheMisses = new AtomicLong();
109     private final AtomicLong cacheUpdates = new AtomicLong();
110 
111     private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
112 
113     private final CacheConfig cacheConfig;
114     private final ClientExecChain backend;
115     private final HttpCache responseCache;
116     private final CacheValidityPolicy validityPolicy;
117     private final CachedHttpResponseGenerator responseGenerator;
118     private final CacheableRequestPolicy cacheableRequestPolicy;
119     private final CachedResponseSuitabilityChecker suitabilityChecker;
120     private final ConditionalRequestBuilder conditionalRequestBuilder;
121     private final ResponseProtocolCompliance responseCompliance;
122     private final RequestProtocolCompliance requestCompliance;
123     private final ResponseCachingPolicy responseCachingPolicy;
124 
125     private final AsynchronousValidator asynchRevalidator;
126 
127     private final Log log = LogFactory.getLog(getClass());
128 
129     public CachingExec(
130             final ClientExecChain backend,
131             final HttpCache cache,
132             final CacheConfig config) {
133         this(backend, cache, config, null);
134     }
135 
136     public CachingExec(
137             final ClientExecChain backend,
138             final HttpCache cache,
139             final CacheConfig config,
140             final AsynchronousValidator asynchRevalidator) {
141         super();
142         Args.notNull(backend, "HTTP backend");
143         Args.notNull(cache, "HttpCache");
144         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
145         this.backend = backend;
146         this.responseCache = cache;
147         this.validityPolicy = new CacheValidityPolicy();
148         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
149         this.cacheableRequestPolicy = new CacheableRequestPolicy();
150         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, this.cacheConfig);
151         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
152         this.responseCompliance = new ResponseProtocolCompliance();
153         this.requestCompliance = new RequestProtocolCompliance(this.cacheConfig.isWeakETagOnPutDeleteAllowed());
154         this.responseCachingPolicy = new ResponseCachingPolicy(
155                 this.cacheConfig.getMaxObjectSize(), this.cacheConfig.isSharedCache(),
156                 this.cacheConfig.isNeverCacheHTTP10ResponsesWithQuery(), this.cacheConfig.is303CachingEnabled());
157         this.asynchRevalidator = asynchRevalidator;
158     }
159 
160     public CachingExec(
161             final ClientExecChain backend,
162             final ResourceFactory resourceFactory,
163             final HttpCacheStorage storage,
164             final CacheConfig config) {
165         this(backend, new BasicHttpCache(resourceFactory, storage, config), config);
166     }
167 
168     public CachingExec(final ClientExecChain backend) {
169         this(backend, new BasicHttpCache(), CacheConfig.DEFAULT);
170     }
171 
172     CachingExec(
173             final ClientExecChain backend,
174             final HttpCache responseCache,
175             final CacheValidityPolicy validityPolicy,
176             final ResponseCachingPolicy responseCachingPolicy,
177             final CachedHttpResponseGenerator responseGenerator,
178             final CacheableRequestPolicy cacheableRequestPolicy,
179             final CachedResponseSuitabilityChecker suitabilityChecker,
180             final ConditionalRequestBuilder conditionalRequestBuilder,
181             final ResponseProtocolCompliance responseCompliance,
182             final RequestProtocolCompliance requestCompliance,
183             final CacheConfig config,
184             final AsynchronousValidator asynchRevalidator) {
185         this.cacheConfig = config != null ? config : CacheConfig.DEFAULT;
186         this.backend = backend;
187         this.responseCache = responseCache;
188         this.validityPolicy = validityPolicy;
189         this.responseCachingPolicy = responseCachingPolicy;
190         this.responseGenerator = responseGenerator;
191         this.cacheableRequestPolicy = cacheableRequestPolicy;
192         this.suitabilityChecker = suitabilityChecker;
193         this.conditionalRequestBuilder = conditionalRequestBuilder;
194         this.responseCompliance = responseCompliance;
195         this.requestCompliance = requestCompliance;
196         this.asynchRevalidator = asynchRevalidator;
197     }
198 
199     /**
200      * Reports the number of times that the cache successfully responded
201      * to an {@link HttpRequest} without contacting the origin server.
202      * @return the number of cache hits
203      */
204     public long getCacheHits() {
205         return cacheHits.get();
206     }
207 
208     /**
209      * Reports the number of times that the cache contacted the origin
210      * server because it had no appropriate response cached.
211      * @return the number of cache misses
212      */
213     public long getCacheMisses() {
214         return cacheMisses.get();
215     }
216 
217     /**
218      * Reports the number of times that the cache was able to satisfy
219      * a response by revalidating an existing but stale cache entry.
220      * @return the number of cache revalidations
221      */
222     public long getCacheUpdates() {
223         return cacheUpdates.get();
224     }
225 
226     public CloseableHttpResponse execute(
227             final HttpRoute route,
228             final HttpRequestWrapper request) throws IOException, HttpException {
229         return execute(route, request, HttpClientContext.create(), null);
230     }
231 
232     public CloseableHttpResponse execute(
233             final HttpRoute route,
234             final HttpRequestWrapper request,
235             final HttpClientContext context) throws IOException, HttpException {
236         return execute(route, request, context, null);
237     }
238 
239     @Override
240     public CloseableHttpResponse execute(
241             final HttpRoute route,
242             final HttpRequestWrapper request,
243             final HttpClientContext context,
244             final HttpExecutionAware execAware) throws IOException, HttpException {
245 
246         final HttpHost target = context.getTargetHost();
247         final String via = generateViaHeader(request.getOriginal());
248 
249         // default response context
250         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
251 
252         if (clientRequestsOurOptions(request)) {
253             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
254             return Proxies.enhanceResponse(new OptionsHttp11Response());
255         }
256 
257         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(request, context);
258         if (fatalErrorResponse != null) {
259             return Proxies.enhanceResponse(fatalErrorResponse);
260         }
261 
262         requestCompliance.makeRequestCompliant(request);
263         request.addHeader("Via",via);
264 
265         flushEntriesInvalidatedByRequest(context.getTargetHost(), request);
266 
267         if (!cacheableRequestPolicy.isServableFromCache(request)) {
268             log.debug("Request is not servable from cache");
269             return callBackend(route, request, context, execAware);
270         }
271 
272         final HttpCacheEntry entry = satisfyFromCache(target, request);
273         if (entry == null) {
274             log.debug("Cache miss");
275             return handleCacheMiss(route, request, context, execAware);
276         } else {
277             return handleCacheHit(route, request, context, execAware, entry);
278         }
279     }
280 
281     private CloseableHttpResponse handleCacheHit(
282             final HttpRoute route,
283             final HttpRequestWrapper request,
284             final HttpClientContext context,
285             final HttpExecutionAware execAware,
286             final HttpCacheEntry entry) throws IOException, HttpException {
287         final HttpHost target = context.getTargetHost();
288         recordCacheHit(target, request);
289         CloseableHttpResponse out = null;
290         final Date now = getCurrentDate();
291         if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
292             log.debug("Cache hit");
293             out = generateCachedResponse(request, context, entry, now);
294         } else if (!mayCallBackend(request)) {
295             log.debug("Cache entry not suitable but only-if-cached requested");
296             out = generateGatewayTimeout(context);
297         } else if (!(entry.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
298                 && !suitabilityChecker.isConditional(request))) {
299             log.debug("Revalidating cache entry");
300             return revalidateCacheEntry(route, request, context, execAware, entry, now);
301         } else {
302             log.debug("Cache entry not usable; calling backend");
303             return callBackend(route, request, context, execAware);
304         }
305         context.setAttribute(HttpClientContext.HTTP_ROUTE, route);
306         context.setAttribute(HttpCoreContext.HTTP_TARGET_HOST, target);
307         context.setAttribute(HttpCoreContext.HTTP_REQUEST, request);
308         context.setAttribute(HttpCoreContext.HTTP_RESPONSE, out);
309         context.setAttribute(HttpCoreContext.HTTP_REQ_SENT, Boolean.TRUE);
310         return out;
311     }
312 
313     private CloseableHttpResponse revalidateCacheEntry(
314             final HttpRoute route,
315             final HttpRequestWrapper request,
316             final HttpClientContext context,
317             final HttpExecutionAware execAware,
318             final HttpCacheEntry entry,
319             final Date now) throws HttpException {
320 
321         try {
322             if (asynchRevalidator != null
323                 && !staleResponseNotAllowed(request, entry, now)
324                 && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
325                 log.trace("Serving stale with asynchronous revalidation");
326                 final CloseableHttpResponse resp = generateCachedResponse(request, context, entry, now);
327                 asynchRevalidator.revalidateCacheEntry(this, route, request, context, execAware, entry);
328                 return resp;
329             }
330             return revalidateCacheEntry(route, request, context, execAware, entry);
331         } catch (final IOException ioex) {
332             return handleRevalidationFailure(request, context, entry, now);
333         }
334     }
335 
336     private CloseableHttpResponse handleCacheMiss(
337             final HttpRoute route,
338             final HttpRequestWrapper request,
339             final HttpClientContext context,
340             final HttpExecutionAware execAware) throws IOException, HttpException {
341         final HttpHost target = context.getTargetHost();
342         recordCacheMiss(target, request);
343 
344         if (!mayCallBackend(request)) {
345             return Proxies.enhanceResponse(
346                     new BasicHttpResponse(
347                             HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout"));
348         }
349 
350         final Map<String, Variant> variants = getExistingCacheVariants(target, request);
351         if (variants != null && !variants.isEmpty()) {
352             return negotiateResponseFromVariants(route, request, context,
353                     execAware, variants);
354         }
355 
356         return callBackend(route, request, context, execAware);
357     }
358 
359     private HttpCacheEntry satisfyFromCache(
360             final HttpHost target, final HttpRequestWrapper request) {
361         HttpCacheEntry entry = null;
362         try {
363             entry = responseCache.getCacheEntry(target, request);
364         } catch (final IOException ioe) {
365             log.warn("Unable to retrieve entries from cache", ioe);
366         }
367         return entry;
368     }
369 
370     private HttpResponse getFatallyNoncompliantResponse(
371             final HttpRequestWrapper request,
372             final HttpContext context) {
373         HttpResponse fatalErrorResponse = null;
374         final List<RequestProtocolError> fatalError = requestCompliance.requestIsFatallyNonCompliant(request);
375 
376         for (final RequestProtocolError error : fatalError) {
377             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
378             fatalErrorResponse = requestCompliance.getErrorForRequest(error);
379         }
380         return fatalErrorResponse;
381     }
382 
383     private Map<String, Variant> getExistingCacheVariants(
384             final HttpHost target,
385             final HttpRequestWrapper request) {
386         Map<String,Variant> variants = null;
387         try {
388             variants = responseCache.getVariantCacheEntriesWithEtags(target, request);
389         } catch (final IOException ioe) {
390             log.warn("Unable to retrieve variant entries from cache", ioe);
391         }
392         return variants;
393     }
394 
395     private void recordCacheMiss(final HttpHost target, final HttpRequestWrapper request) {
396         cacheMisses.getAndIncrement();
397         if (log.isTraceEnabled()) {
398             final RequestLine rl = request.getRequestLine();
399             log.trace("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
400         }
401     }
402 
403     private void recordCacheHit(final HttpHost target, final HttpRequestWrapper request) {
404         cacheHits.getAndIncrement();
405         if (log.isTraceEnabled()) {
406             final RequestLine rl = request.getRequestLine();
407             log.trace("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
408         }
409     }
410 
411     private void recordCacheUpdate(final HttpContext context) {
412         cacheUpdates.getAndIncrement();
413         setResponseStatus(context, CacheResponseStatus.VALIDATED);
414     }
415 
416     private void flushEntriesInvalidatedByRequest(
417             final HttpHost target,
418             final HttpRequestWrapper request) {
419         try {
420             responseCache.flushInvalidatedCacheEntriesFor(target, request);
421         } catch (final IOException ioe) {
422             log.warn("Unable to flush invalidated entries from cache", ioe);
423         }
424     }
425 
426     private CloseableHttpResponse generateCachedResponse(final HttpRequestWrapper request,
427             final HttpContext context, final HttpCacheEntry entry, final Date now) {
428         final CloseableHttpResponse cachedResponse;
429         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
430                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
431             cachedResponse = responseGenerator.generateNotModifiedResponse(entry);
432         } else {
433             cachedResponse = responseGenerator.generateResponse(entry);
434         }
435         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
436         if (validityPolicy.getStalenessSecs(entry, now) > 0L) {
437             cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\"");
438         }
439         return cachedResponse;
440     }
441 
442     private CloseableHttpResponse handleRevalidationFailure(
443             final HttpRequestWrapper request,
444             final HttpContext context,
445             final HttpCacheEntry entry,
446             final Date now) {
447         if (staleResponseNotAllowed(request, entry, now)) {
448             return generateGatewayTimeout(context);
449         } else {
450             return unvalidatedCacheHit(context, entry);
451         }
452     }
453 
454     private CloseableHttpResponse generateGatewayTimeout(
455             final HttpContext context) {
456         setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
457         return Proxies.enhanceResponse(new BasicHttpResponse(
458                 HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT,
459                 "Gateway Timeout"));
460     }
461 
462     private CloseableHttpResponse unvalidatedCacheHit(
463             final HttpContext context, final HttpCacheEntry entry) {
464         final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(entry);
465         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
466         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
467         return cachedResponse;
468     }
469 
470     private boolean staleResponseNotAllowed(
471             final HttpRequestWrapper request,
472             final HttpCacheEntry entry,
473             final Date now) {
474         return validityPolicy.mustRevalidate(entry)
475             || (cacheConfig.isSharedCache() && validityPolicy.proxyRevalidate(entry))
476             || explicitFreshnessRequest(request, entry, now);
477     }
478 
479     private boolean mayCallBackend(final HttpRequestWrapper request) {
480         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
481             for (final HeaderElement elt : h.getElements()) {
482                 if ("only-if-cached".equals(elt.getName())) {
483                     log.trace("Request marked only-if-cached");
484                     return false;
485                 }
486             }
487         }
488         return true;
489     }
490 
491     private boolean explicitFreshnessRequest(
492             final HttpRequestWrapper request,
493             final HttpCacheEntry entry,
494             final Date now) {
495         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
496             for(final HeaderElement elt : h.getElements()) {
497                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
498                     try {
499                         final int maxstale = Integer.parseInt(elt.getValue());
500                         final long age = validityPolicy.getCurrentAgeSecs(entry, now);
501                         final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry);
502                         if (age - lifetime > maxstale) {
503                             return true;
504                         }
505                     } catch (final NumberFormatException nfe) {
506                         return true;
507                     }
508                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
509                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
510                     return true;
511                 }
512             }
513         }
514         return false;
515     }
516 
517     private String generateViaHeader(final HttpMessage msg) {
518 
519         final ProtocolVersion pv = msg.getProtocolVersion();
520         final String existingEntry = viaHeaders.get(pv);
521         if (existingEntry != null) {
522             return existingEntry;
523         }
524 
525         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
526         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
527 
528         String value;
529         final Integer major = Integer.valueOf(pv.getMajor());
530         final Integer minor = Integer.valueOf(pv.getMinor());
531         if ("http".equalsIgnoreCase(pv.getProtocol())) {
532             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", major, minor,
533                     release);
534         } else {
535             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), major,
536                     minor, release);
537         }
538         viaHeaders.put(pv, value);
539 
540         return value;
541     }
542 
543     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
544         if (context != null) {
545             context.setAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS, value);
546         }
547     }
548 
549     /**
550      * Reports whether this {@code CachingHttpClient} implementation
551      * supports byte-range requests as specified by the {@code Range}
552      * and {@code Content-Range} headers.
553      * @return {@code true} if byte-range requests are supported
554      */
555     public boolean supportsRangeAndContentRangeHeaders() {
556         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
557     }
558 
559     Date getCurrentDate() {
560         return new Date();
561     }
562 
563     boolean clientRequestsOurOptions(final HttpRequest request) {
564         final RequestLine line = request.getRequestLine();
565 
566         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
567             return false;
568         }
569 
570         if (!"*".equals(line.getUri())) {
571             return false;
572         }
573 
574         if (!"0".equals(request.getFirstHeader(HeaderConstants.MAX_FORWARDS).getValue())) {
575             return false;
576         }
577 
578         return true;
579     }
580 
581     CloseableHttpResponse callBackend(
582             final HttpRoute route,
583             final HttpRequestWrapper request,
584             final HttpClientContext context,
585             final HttpExecutionAware execAware) throws IOException, HttpException  {
586 
587         final Date requestDate = getCurrentDate();
588 
589         log.trace("Calling the backend");
590         final CloseableHttpResponse backendResponse = backend.execute(route, request, context, execAware);
591         try {
592             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
593             return handleBackendResponse(request, context, requestDate, getCurrentDate(),
594                     backendResponse);
595         } catch (final IOException ex) {
596             backendResponse.close();
597             throw ex;
598         } catch (final RuntimeException ex) {
599             backendResponse.close();
600             throw ex;
601         }
602     }
603 
604     private boolean revalidationResponseIsTooOld(final HttpResponse backendResponse,
605             final HttpCacheEntry cacheEntry) {
606         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
607         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
608         if (entryDateHeader != null && responseDateHeader != null) {
609             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
610             final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
611             if (entryDate == null || respDate == null) {
612                 // either backend response or cached entry did not have a valid
613                 // Date header, so we can't tell if they are out of order
614                 // according to the origin clock; thus we can skip the
615                 // unconditional retry recommended in 13.2.6 of RFC 2616.
616                 return false;
617             }
618             if (respDate.before(entryDate)) {
619                 return true;
620             }
621         }
622         return false;
623     }
624 
625     CloseableHttpResponse negotiateResponseFromVariants(
626             final HttpRoute route,
627             final HttpRequestWrapper request,
628             final HttpClientContext context,
629             final HttpExecutionAware execAware,
630             final Map<String, Variant> variants) throws IOException, HttpException {
631         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder
632             .buildConditionalRequestFromVariants(request, variants);
633 
634         final Date requestDate = getCurrentDate();
635         final CloseableHttpResponse backendResponse = backend.execute(
636                 route, conditionalRequest, context, execAware);
637         try {
638             final Date responseDate = getCurrentDate();
639 
640             backendResponse.addHeader("Via", generateViaHeader(backendResponse));
641 
642             if (backendResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
643                 return handleBackendResponse(request, context, requestDate, responseDate,
644                         backendResponse);
645             }
646 
647             final Header resultEtagHeader = backendResponse.getFirstHeader(HeaderConstants.ETAG);
648             if (resultEtagHeader == null) {
649                 log.warn("304 response did not contain ETag");
650                 IOUtils.consume(backendResponse.getEntity());
651                 backendResponse.close();
652                 return callBackend(route, request, context, execAware);
653             }
654 
655             final String resultEtag = resultEtagHeader.getValue();
656             final Variant matchingVariant = variants.get(resultEtag);
657             if (matchingVariant == null) {
658                 log.debug("304 response did not contain ETag matching one sent in If-None-Match");
659                 IOUtils.consume(backendResponse.getEntity());
660                 backendResponse.close();
661                 return callBackend(route, request, context, execAware);
662             }
663 
664             final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
665 
666             if (revalidationResponseIsTooOld(backendResponse, matchedEntry)) {
667                 IOUtils.consume(backendResponse.getEntity());
668                 backendResponse.close();
669                 return retryRequestUnconditionally(route, request, context, execAware, matchedEntry);
670             }
671 
672             recordCacheUpdate(context);
673 
674             final HttpCacheEntry responseEntry = getUpdatedVariantEntry(
675                 context.getTargetHost(), conditionalRequest, requestDate, responseDate,
676                     backendResponse, matchingVariant, matchedEntry);
677             backendResponse.close();
678 
679             final CloseableHttpResponse resp = responseGenerator.generateResponse(responseEntry);
680             tryToUpdateVariantMap(context.getTargetHost(), request, matchingVariant);
681 
682             if (shouldSendNotModifiedResponse(request, responseEntry)) {
683                 return responseGenerator.generateNotModifiedResponse(responseEntry);
684             }
685             return resp;
686         } catch (final IOException ex) {
687             backendResponse.close();
688             throw ex;
689         } catch (final RuntimeException ex) {
690             backendResponse.close();
691             throw ex;
692         }
693     }
694 
695     private CloseableHttpResponse retryRequestUnconditionally(
696             final HttpRoute route,
697             final HttpRequestWrapper request,
698             final HttpClientContext context,
699             final HttpExecutionAware execAware,
700             final HttpCacheEntry matchedEntry) throws IOException, HttpException {
701         final HttpRequestWrapper unconditional = conditionalRequestBuilder
702             .buildUnconditionalRequest(request, matchedEntry);
703         return callBackend(route, unconditional, context, execAware);
704     }
705 
706     private HttpCacheEntry getUpdatedVariantEntry(
707             final HttpHost target,
708             final HttpRequestWrapper conditionalRequest,
709             final Date requestDate,
710             final Date responseDate,
711             final CloseableHttpResponse backendResponse,
712             final Variant matchingVariant,
713             final HttpCacheEntry matchedEntry) throws IOException {
714         HttpCacheEntry responseEntry = matchedEntry;
715         try {
716             responseEntry = responseCache.updateVariantCacheEntry(target, conditionalRequest,
717                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
718         } catch (final IOException ioe) {
719             log.warn("Could not update cache entry", ioe);
720         } finally {
721             backendResponse.close();
722         }
723         return responseEntry;
724     }
725 
726     private void tryToUpdateVariantMap(
727             final HttpHost target,
728             final HttpRequestWrapper request,
729             final Variant matchingVariant) {
730         try {
731             responseCache.reuseVariantEntryFor(target, request, matchingVariant);
732         } catch (final IOException ioe) {
733             log.warn("Could not update cache entry to reuse variant", ioe);
734         }
735     }
736 
737     private boolean shouldSendNotModifiedResponse(
738             final HttpRequestWrapper request,
739             final HttpCacheEntry responseEntry) {
740         return (suitabilityChecker.isConditional(request)
741                 && suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
742     }
743 
744     CloseableHttpResponse revalidateCacheEntry(
745             final HttpRoute route,
746             final HttpRequestWrapper request,
747             final HttpClientContext context,
748             final HttpExecutionAware execAware,
749             final HttpCacheEntry cacheEntry) throws IOException, HttpException {
750 
751         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
752         final URI uri = conditionalRequest.getURI();
753         if (uri != null) {
754             try {
755                 conditionalRequest.setURI(URIUtils.rewriteURIForRoute(uri, route));
756             } catch (final URISyntaxException ex) {
757                 throw new ProtocolException("Invalid URI: " + uri, ex);
758             }
759         }
760 
761         Date requestDate = getCurrentDate();
762         CloseableHttpResponse backendResponse = backend.execute(
763                 route, conditionalRequest, context, execAware);
764         Date responseDate = getCurrentDate();
765 
766         if (revalidationResponseIsTooOld(backendResponse, cacheEntry)) {
767             backendResponse.close();
768             final HttpRequestWrapper unconditional = conditionalRequestBuilder
769                 .buildUnconditionalRequest(request, cacheEntry);
770             requestDate = getCurrentDate();
771             backendResponse = backend.execute(route, unconditional, context, execAware);
772             responseDate = getCurrentDate();
773         }
774 
775         backendResponse.addHeader(HeaderConstants.VIA, generateViaHeader(backendResponse));
776 
777         final int statusCode = backendResponse.getStatusLine().getStatusCode();
778         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
779             recordCacheUpdate(context);
780         }
781 
782         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
783             final HttpCacheEntry updatedEntry = responseCache.updateCacheEntry(
784                     context.getTargetHost(), request, cacheEntry,
785                     backendResponse, requestDate, responseDate);
786             if (suitabilityChecker.isConditional(request)
787                     && suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
788                 return responseGenerator
789                         .generateNotModifiedResponse(updatedEntry);
790             }
791             return responseGenerator.generateResponse(updatedEntry);
792         }
793 
794         if (staleIfErrorAppliesTo(statusCode)
795             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
796             && validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
797             try {
798                 final CloseableHttpResponse cachedResponse = responseGenerator.generateResponse(cacheEntry);
799                 cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
800                 return cachedResponse;
801             } finally {
802                 backendResponse.close();
803             }
804         }
805         return handleBackendResponse(conditionalRequest, context, requestDate, responseDate,
806                 backendResponse);
807     }
808 
809     private boolean staleIfErrorAppliesTo(final int statusCode) {
810         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
811                 || statusCode == HttpStatus.SC_BAD_GATEWAY
812                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
813                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
814     }
815 
816     CloseableHttpResponse handleBackendResponse(
817             final HttpRequestWrapper request,
818             final HttpClientContext context,
819             final Date requestDate,
820             final Date responseDate,
821             final CloseableHttpResponse backendResponse) throws IOException {
822 
823         log.trace("Handling Backend response");
824         responseCompliance.ensureProtocolCompliance(request, backendResponse);
825 
826         final HttpHost target = context.getTargetHost();
827         final boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse);
828         responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
829         if (cacheable && !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
830             storeRequestIfModifiedSinceFor304Response(request, backendResponse);
831             return responseCache.cacheAndReturnResponse(target, request,
832                     backendResponse, requestDate, responseDate);
833         }
834         if (!cacheable) {
835             try {
836                 responseCache.flushCacheEntriesFor(target, request);
837             } catch (final IOException ioe) {
838                 log.warn("Unable to flush invalid cache entries", ioe);
839             }
840         }
841         return backendResponse;
842     }
843 
844     /**
845      * For 304 Not modified responses, adds a "Last-Modified" header with the
846      * value of the "If-Modified-Since" header passed in the request. This
847      * header is required to be able to reuse match the cache entry for
848      * subsequent requests but as defined in http specifications it is not
849      * included in 304 responses by backend servers. This header will not be
850      * included in the resulting response.
851      */
852     private void storeRequestIfModifiedSinceFor304Response(
853             final HttpRequest request, final HttpResponse backendResponse) {
854         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
855             final Header h = request.getFirstHeader("If-Modified-Since");
856             if (h != null) {
857                 backendResponse.addHeader("Last-Modified", h.getValue());
858             }
859         }
860     }
861 
862     private boolean alreadyHaveNewerCacheEntry(final HttpHost target, final HttpRequestWrapper request,
863             final HttpResponse backendResponse) {
864         HttpCacheEntry existing = null;
865         try {
866             existing = responseCache.getCacheEntry(target, request);
867         } catch (final IOException ioe) {
868             // nop
869         }
870         if (existing == null) {
871             return false;
872         }
873         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
874         if (entryDateHeader == null) {
875             return false;
876         }
877         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
878         if (responseDateHeader == null) {
879             return false;
880         }
881         final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
882         final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
883         if (entryDate == null || responseDate == null) {
884             return false;
885         }
886         return responseDate.before(entryDate);
887     }
888 
889 }