View Javadoc
1   /*
2    * ====================================================================
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *   http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing,
14   * software distributed under the License is distributed on an
15   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16   * KIND, either express or implied.  See the License for the
17   * specific language governing permissions and limitations
18   * under the License.
19   * ====================================================================
20   *
21   * This software consists of voluntary contributions made by many
22   * individuals on behalf of the Apache Software Foundation.  For more
23   * information on the Apache Software Foundation, please see
24   * <http://www.apache.org/>.
25   *
26   */
27  package org.apache.http.impl.client.cache;
28  
29  import java.io.IOException;
30  import java.lang.reflect.UndeclaredThrowableException;
31  import java.net.URI;
32  import java.util.Collections;
33  import java.util.Date;
34  import java.util.HashMap;
35  import java.util.HashSet;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Set;
39  import java.util.concurrent.ArrayBlockingQueue;
40  import java.util.concurrent.ExecutorService;
41  import java.util.concurrent.RejectedExecutionException;
42  import java.util.concurrent.ThreadPoolExecutor;
43  import java.util.concurrent.TimeUnit;
44  import java.util.concurrent.atomic.AtomicLong;
45  
46  import org.apache.commons.logging.Log;
47  import org.apache.commons.logging.LogFactory;
48  import org.apache.http.Header;
49  import org.apache.http.HeaderElement;
50  import org.apache.http.HttpEntity;
51  import org.apache.http.HttpHost;
52  import org.apache.http.HttpMessage;
53  import org.apache.http.HttpRequest;
54  import org.apache.http.HttpResponse;
55  import org.apache.http.HttpStatus;
56  import org.apache.http.HttpVersion;
57  import org.apache.http.ProtocolException;
58  import org.apache.http.ProtocolVersion;
59  import org.apache.http.RequestLine;
60  import org.apache.http.annotation.Contract;
61  import org.apache.http.annotation.ThreadingBehavior;
62  import org.apache.http.client.ClientProtocolException;
63  import org.apache.http.client.HttpClient;
64  import org.apache.http.client.ResponseHandler;
65  import org.apache.http.client.cache.CacheResponseStatus;
66  import org.apache.http.client.cache.HeaderConstants;
67  import org.apache.http.client.cache.HttpCacheEntry;
68  import org.apache.http.client.cache.HttpCacheStorage;
69  import org.apache.http.client.cache.ResourceFactory;
70  import org.apache.http.client.methods.HttpRequestWrapper;
71  import org.apache.http.client.methods.HttpUriRequest;
72  import org.apache.http.conn.ClientConnectionManager;
73  import org.apache.http.impl.client.DefaultHttpClient;
74  import org.apache.http.impl.cookie.DateParseException;
75  import org.apache.http.impl.cookie.DateUtils;
76  import org.apache.http.message.BasicHttpResponse;
77  import org.apache.http.params.HttpParams;
78  import org.apache.http.protocol.ExecutionContext;
79  import org.apache.http.protocol.HTTP;
80  import org.apache.http.protocol.HttpContext;
81  import org.apache.http.util.Args;
82  import org.apache.http.util.VersionInfo;
83  
84  /**
85   * <p>
86   * The {@link CachingHttpClient} is meant to be a drop-in replacement for
87   * a {@link DefaultHttpClient} that transparently adds client-side caching.
88   * The current implementation is conditionally compliant with HTTP/1.1
89   * (meaning all the MUST and MUST NOTs are obeyed), although quite a lot,
90   * though not all, of the SHOULDs and SHOULD NOTs are obeyed too. Generally
91   * speaking, you construct a {@code CachingHttpClient} by providing a
92   * "backend" {@link HttpClient} used for making actual network requests and
93   * provide an {@link HttpCacheStorage} instance to use for holding onto
94   * cached responses. Additional configuration options can be provided by
95   * passing in a {@link CacheConfig}. Note that all of the usual client
96   * related configuration you want to do vis-a-vis timeouts and connection
97   * pools should be done on this backend client before constructing a {@code
98   * CachingHttpClient} from it.
99   * </p>
100  *
101  * <p>
102  * Generally speaking, the {@code CachingHttpClient} is implemented as a
103  * <a href="http://en.wikipedia.org/wiki/Decorator_pattern">Decorator</a>
104  * of the backend client; for any incoming request it attempts to satisfy
105  * it from the cache, but if it can't, or if it needs to revalidate a stale
106  * cache entry, it will use the backend client to make an actual request.
107  * However, a proper HTTP/1.1 cache won't change the semantics of a request
108  * and response; in particular, if you issue an unconditional request you
109  * will get a full response (although it may be served to you from the cache,
110  * or the cache may make a conditional request on your behalf to the origin).
111  * This notion of "semantic transparency" means you should be able to drop
112  * a {@link CachingHttpClient} into an existing application without breaking
113  * anything.
114  * </p>
115  *
116  * <p>
117  * Folks that would like to experiment with alternative storage backends
118  * should look at the {@link HttpCacheStorage} interface and the related
119  * package documentation there. You may also be interested in the provided
120  * {@link org.apache.http.impl.client.cache.ehcache.EhcacheHttpCacheStorage
121  * EhCache} and {@link
122  * org.apache.http.impl.client.cache.memcached.MemcachedHttpCacheStorage
123  * memcached} storage backends.
124  * </p>
125  * @since 4.1
126  *
127  * @deprecated (4.3) use {@link CachingHttpClientBuilder} or {@link CachingHttpClients}.
128  */
129 @Deprecated
130 @Contract(threading = ThreadingBehavior.SAFE_CONDITIONAL)
131 public class CachingHttpClient implements HttpClient {
132 
133     /**
134      * This is the name under which the {@link
135      * org.apache.http.client.cache.CacheResponseStatus} of a request
136      * (for example, whether it resulted in a cache hit) will be recorded if an
137      * {@link HttpContext} is provided during execution.
138      */
139     public static final String CACHE_RESPONSE_STATUS = "http.cache.response.status";
140 
141     private final static boolean SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS = false;
142 
143     private final AtomicLong cacheHits = new AtomicLong();
144     private final AtomicLong cacheMisses = new AtomicLong();
145     private final AtomicLong cacheUpdates = new AtomicLong();
146 
147     private final Map<ProtocolVersion, String> viaHeaders = new HashMap<ProtocolVersion, String>(4);
148 
149     private final HttpClient backend;
150     private final HttpCache responseCache;
151     private final CacheValidityPolicy validityPolicy;
152     private final ResponseCachingPolicy responseCachingPolicy;
153     private final CachedHttpResponseGenerator responseGenerator;
154     private final CacheableRequestPolicy cacheableRequestPolicy;
155     private final CachedResponseSuitabilityChecker suitabilityChecker;
156 
157     private final ConditionalRequestBuilder conditionalRequestBuilder;
158 
159     private final long maxObjectSizeBytes;
160     private final boolean sharedCache;
161 
162     private final ResponseProtocolCompliance responseCompliance;
163     private final RequestProtocolCompliance requestCompliance;
164 
165     private final AsynchronousValidator asynchRevalidator;
166 
167     private final Log log = LogFactory.getLog(getClass());
168 
169     CachingHttpClient(
170             final HttpClient client,
171             final HttpCache cache,
172             final CacheConfig config) {
173         super();
174         Args.notNull(client, "HttpClient");
175         Args.notNull(cache, "HttpCache");
176         Args.notNull(config, "CacheConfig");
177         this.maxObjectSizeBytes = config.getMaxObjectSize();
178         this.sharedCache = config.isSharedCache();
179         this.backend = client;
180         this.responseCache = cache;
181         this.validityPolicy = new CacheValidityPolicy();
182         this.responseCachingPolicy = new ResponseCachingPolicy(maxObjectSizeBytes, sharedCache,
183                 config.isNeverCacheHTTP10ResponsesWithQuery(), config.is303CachingEnabled());
184         this.responseGenerator = new CachedHttpResponseGenerator(this.validityPolicy);
185         this.cacheableRequestPolicy = new CacheableRequestPolicy();
186         this.suitabilityChecker = new CachedResponseSuitabilityChecker(this.validityPolicy, config);
187         this.conditionalRequestBuilder = new ConditionalRequestBuilder();
188 
189         this.responseCompliance = new ResponseProtocolCompliance();
190         this.requestCompliance = new RequestProtocolCompliance(config.isWeakETagOnPutDeleteAllowed());
191 
192         this.asynchRevalidator = makeAsynchronousValidator(config);
193     }
194 
195     /**
196      * Constructs a {@code CachingHttpClient} with default caching settings that
197      * stores cache entries in memory and uses a vanilla {@link DefaultHttpClient}
198      * for backend requests.
199      */
200     public CachingHttpClient() {
201         this(new DefaultHttpClient(),
202                 new BasicHttpCache(),
203                 new CacheConfig());
204     }
205 
206     /**
207      * Constructs a {@code CachingHttpClient} with the given caching options that
208      * stores cache entries in memory and uses a vanilla {@link DefaultHttpClient}
209      * for backend requests.
210      * @param config cache module options
211      */
212     public CachingHttpClient(final CacheConfig config) {
213         this(new DefaultHttpClient(),
214                 new BasicHttpCache(config),
215                 config);
216     }
217 
218     /**
219      * Constructs a {@code CachingHttpClient} with default caching settings that
220      * stores cache entries in memory and uses the given {@link HttpClient}
221      * for backend requests.
222      * @param client used to make origin requests
223      */
224     public CachingHttpClient(final HttpClient client) {
225         this(client,
226                 new BasicHttpCache(),
227                 new CacheConfig());
228     }
229 
230     /**
231      * Constructs a {@code CachingHttpClient} with the given caching options that
232      * stores cache entries in memory and uses the given {@link HttpClient}
233      * for backend requests.
234      * @param config cache module options
235      * @param client used to make origin requests
236      */
237     public CachingHttpClient(final HttpClient client, final CacheConfig config) {
238         this(client,
239                 new BasicHttpCache(config),
240                 config);
241     }
242 
243     /**
244      * Constructs a {@code CachingHttpClient} with the given caching options
245      * that stores cache entries in the provided storage backend and uses
246      * the given {@link HttpClient} for backend requests. However, cached
247      * response bodies are managed using the given {@link ResourceFactory}.
248      * @param client used to make origin requests
249      * @param resourceFactory how to manage cached response bodies
250      * @param storage where to store cache entries
251      * @param config cache module options
252      */
253     public CachingHttpClient(
254             final HttpClient client,
255             final ResourceFactory resourceFactory,
256             final HttpCacheStorage storage,
257             final CacheConfig config) {
258         this(client,
259                 new BasicHttpCache(resourceFactory, storage, config),
260                 config);
261     }
262 
263     /**
264      * Constructs a {@code CachingHttpClient} with the given caching options
265      * that stores cache entries in the provided storage backend and uses
266      * the given {@link HttpClient} for backend requests.
267      * @param client used to make origin requests
268      * @param storage where to store cache entries
269      * @param config cache module options
270      */
271     public CachingHttpClient(
272             final HttpClient client,
273             final HttpCacheStorage storage,
274             final CacheConfig config) {
275         this(client,
276                 new BasicHttpCache(new HeapResourceFactory(), storage, config),
277                 config);
278     }
279 
280     CachingHttpClient(
281             final HttpClient backend,
282             final CacheValidityPolicy validityPolicy,
283             final ResponseCachingPolicy responseCachingPolicy,
284             final HttpCache responseCache,
285             final CachedHttpResponseGenerator responseGenerator,
286             final CacheableRequestPolicy cacheableRequestPolicy,
287             final CachedResponseSuitabilityChecker suitabilityChecker,
288             final ConditionalRequestBuilder conditionalRequestBuilder,
289             final ResponseProtocolCompliance responseCompliance,
290             final RequestProtocolCompliance requestCompliance) {
291         final CacheConfig config = new CacheConfig();
292         this.maxObjectSizeBytes = config.getMaxObjectSize();
293         this.sharedCache = config.isSharedCache();
294         this.backend = backend;
295         this.validityPolicy = validityPolicy;
296         this.responseCachingPolicy = responseCachingPolicy;
297         this.responseCache = responseCache;
298         this.responseGenerator = responseGenerator;
299         this.cacheableRequestPolicy = cacheableRequestPolicy;
300         this.suitabilityChecker = suitabilityChecker;
301         this.conditionalRequestBuilder = conditionalRequestBuilder;
302         this.responseCompliance = responseCompliance;
303         this.requestCompliance = requestCompliance;
304         this.asynchRevalidator = makeAsynchronousValidator(config);
305     }
306 
307     private AsynchronousValidator makeAsynchronousValidator(
308             final CacheConfig config) {
309         if (config.getAsynchronousWorkersMax() > 0) {
310             return new AsynchronousValidator(this, config);
311         }
312         return null;
313     }
314 
315     /**
316      * Reports the number of times that the cache successfully responded
317      * to an {@link HttpRequest} without contacting the origin server.
318      * @return the number of cache hits
319      */
320     public long getCacheHits() {
321         return cacheHits.get();
322     }
323 
324     /**
325      * Reports the number of times that the cache contacted the origin
326      * server because it had no appropriate response cached.
327      * @return the number of cache misses
328      */
329     public long getCacheMisses() {
330         return cacheMisses.get();
331     }
332 
333     /**
334      * Reports the number of times that the cache was able to satisfy
335      * a response by revalidating an existing but stale cache entry.
336      * @return the number of cache revalidations
337      */
338     public long getCacheUpdates() {
339         return cacheUpdates.get();
340     }
341 
342     @Override
343     public HttpResponse execute(final HttpHost target, final HttpRequest request) throws IOException {
344         final HttpContext defaultContext = null;
345         return execute(target, request, defaultContext);
346     }
347 
348     @Override
349     public <T> T execute(final HttpHost target, final HttpRequest request,
350                          final ResponseHandler<? extends T> responseHandler) throws IOException {
351         return execute(target, request, responseHandler, null);
352     }
353 
354     @Override
355     public <T> T execute(final HttpHost target, final HttpRequest request,
356                          final ResponseHandler<? extends T> responseHandler, final HttpContext context) throws IOException {
357         final HttpResponse resp = execute(target, request, context);
358         return handleAndConsume(responseHandler,resp);
359     }
360 
361     @Override
362     public HttpResponse execute(final HttpUriRequest request) throws IOException {
363         final HttpContext context = null;
364         return execute(request, context);
365     }
366 
367     @Override
368     public HttpResponse execute(final HttpUriRequest request, final HttpContext context) throws IOException {
369         final URI uri = request.getURI();
370         final HttpHost httpHost = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
371         return execute(httpHost, request, context);
372     }
373 
374     @Override
375     public <T> T execute(final HttpUriRequest request, final ResponseHandler<? extends T> responseHandler)
376             throws IOException {
377         return execute(request, responseHandler, null);
378     }
379 
380     @Override
381     public <T> T execute(final HttpUriRequest request, final ResponseHandler<? extends T> responseHandler,
382                          final HttpContext context) throws IOException {
383         final HttpResponse resp = execute(request, context);
384         return handleAndConsume(responseHandler, resp);
385     }
386 
387     private <T> T handleAndConsume(
388             final ResponseHandler<? extends T> responseHandler,
389             final HttpResponse response) throws Error, IOException {
390         final T result;
391         try {
392             result = responseHandler.handleResponse(response);
393         } catch (final Exception t) {
394             final HttpEntity entity = response.getEntity();
395             try {
396                 IOUtils.consume(entity);
397             } catch (final Exception t2) {
398                 // Log this exception. The original exception is more
399                 // important and will be thrown to the caller.
400                 this.log.warn("Error consuming content after an exception.", t2);
401             }
402             if (t instanceof RuntimeException) {
403                 throw (RuntimeException) t;
404             }
405             if (t instanceof IOException) {
406                 throw (IOException) t;
407             }
408             throw new UndeclaredThrowableException(t);
409         }
410 
411         // Handling the response was successful. Ensure that the content has
412         // been fully consumed.
413         final HttpEntity entity = response.getEntity();
414         IOUtils.consume(entity);
415         return result;
416     }
417 
418     @Override
419     public ClientConnectionManager getConnectionManager() {
420         return backend.getConnectionManager();
421     }
422 
423     @Override
424     public HttpParams getParams() {
425         return backend.getParams();
426     }
427 
428     @Override
429     public HttpResponse execute(final HttpHost target, final HttpRequest originalRequest, final HttpContext context)
430             throws IOException {
431 
432         final HttpRequestWrapper request;
433         if (originalRequest instanceof HttpRequestWrapper) {
434             request = ((HttpRequestWrapper) originalRequest);
435         } else {
436             request = HttpRequestWrapper.wrap(originalRequest);
437         }
438         final String via = generateViaHeader(originalRequest);
439 
440         // default response context
441         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
442 
443         if (clientRequestsOurOptions(request)) {
444             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
445             return new OptionsHttp11Response();
446         }
447 
448         final HttpResponse fatalErrorResponse = getFatallyNoncompliantResponse(
449                 request, context);
450         if (fatalErrorResponse != null) {
451             return fatalErrorResponse;
452         }
453 
454         requestCompliance.makeRequestCompliant(request);
455         request.addHeader("Via",via);
456 
457         flushEntriesInvalidatedByRequest(target, request);
458 
459         if (!cacheableRequestPolicy.isServableFromCache(request)) {
460             log.debug("Request is not servable from cache");
461             return callBackend(target, request, context);
462         }
463 
464         final HttpCacheEntry entry = satisfyFromCache(target, request);
465         if (entry == null) {
466             log.debug("Cache miss");
467             return handleCacheMiss(target, request, context);
468         }
469 
470         return handleCacheHit(target, request, context, entry);
471     }
472 
473     private HttpResponse handleCacheHit(final HttpHost target, final HttpRequestWrapper request,
474             final HttpContext context, final HttpCacheEntry entry)
475             throws ClientProtocolException, IOException {
476         recordCacheHit(target, request);
477         HttpResponse out = null;
478         final Date now = getCurrentDate();
479         if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) {
480             log.debug("Cache hit");
481             out = generateCachedResponse(request, context, entry, now);
482         } else if (!mayCallBackend(request)) {
483             log.debug("Cache entry not suitable but only-if-cached requested");
484             out = generateGatewayTimeout(context);
485         } else {
486             log.debug("Revalidating cache entry");
487             return revalidateCacheEntry(target, request, context, entry, now);
488         }
489         if (context != null) {
490             context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, target);
491             context.setAttribute(ExecutionContext.HTTP_REQUEST, request);
492             context.setAttribute(ExecutionContext.HTTP_RESPONSE, out);
493             context.setAttribute(ExecutionContext.HTTP_REQ_SENT, Boolean.TRUE);
494         }
495         return out;
496     }
497 
498     private HttpResponse revalidateCacheEntry(final HttpHost target,
499             final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry,
500             final Date now) throws ClientProtocolException {
501 
502         try {
503             if (asynchRevalidator != null
504                 && !staleResponseNotAllowed(request, entry, now)
505                 && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) {
506                 log.trace("Serving stale with asynchronous revalidation");
507                 final HttpResponse resp = generateCachedResponse(request, context, entry, now);
508 
509                 asynchRevalidator.revalidateCacheEntry(target, request, context, entry);
510 
511                 return resp;
512             }
513             return revalidateCacheEntry(target, request, context, entry);
514         } catch (final IOException ioex) {
515             return handleRevalidationFailure(request, context, entry, now);
516         } catch (final ProtocolException e) {
517             throw new ClientProtocolException(e);
518         }
519     }
520 
521     private HttpResponse handleCacheMiss(final HttpHost target, final HttpRequestWrapper request,
522             final HttpContext context) throws IOException {
523         recordCacheMiss(target, request);
524 
525         if (!mayCallBackend(request)) {
526             return new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT,
527                     "Gateway Timeout");
528         }
529 
530         final Map<String, Variant> variants =
531             getExistingCacheVariants(target, request);
532         if (variants != null && variants.size() > 0) {
533             return negotiateResponseFromVariants(target, request, context, variants);
534         }
535 
536         return callBackend(target, request, context);
537     }
538 
539     private HttpCacheEntry satisfyFromCache(final HttpHost target, final HttpRequestWrapper request) {
540         HttpCacheEntry entry = null;
541         try {
542             entry = responseCache.getCacheEntry(target, request);
543         } catch (final IOException ioe) {
544             log.warn("Unable to retrieve entries from cache", ioe);
545         }
546         return entry;
547     }
548 
549     private HttpResponse getFatallyNoncompliantResponse(final HttpRequestWrapper request,
550             final HttpContext context) {
551         HttpResponse fatalErrorResponse = null;
552         final List<RequestProtocolError> fatalError = requestCompliance.requestIsFatallyNonCompliant(request);
553 
554         for (final RequestProtocolError error : fatalError) {
555             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
556             fatalErrorResponse = requestCompliance.getErrorForRequest(error);
557         }
558         return fatalErrorResponse;
559     }
560 
561     private Map<String, Variant> getExistingCacheVariants(final HttpHost target,
562             final HttpRequestWrapper request) {
563         Map<String,Variant> variants = null;
564         try {
565             variants = responseCache.getVariantCacheEntriesWithEtags(target, request);
566         } catch (final IOException ioe) {
567             log.warn("Unable to retrieve variant entries from cache", ioe);
568         }
569         return variants;
570     }
571 
572     private void recordCacheMiss(final HttpHost target, final HttpRequestWrapper request) {
573         cacheMisses.getAndIncrement();
574         if (log.isTraceEnabled()) {
575             final RequestLine rl = request.getRequestLine();
576             log.trace("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]");
577         }
578     }
579 
580     private void recordCacheHit(final HttpHost target, final HttpRequestWrapper request) {
581         cacheHits.getAndIncrement();
582         if (log.isTraceEnabled()) {
583             final RequestLine rl = request.getRequestLine();
584             log.trace("Cache hit [host: " + target + "; uri: " + rl.getUri() + "]");
585         }
586     }
587 
588     private void recordCacheUpdate(final HttpContext context) {
589         cacheUpdates.getAndIncrement();
590         setResponseStatus(context, CacheResponseStatus.VALIDATED);
591     }
592 
593     private void flushEntriesInvalidatedByRequest(final HttpHost target,
594             final HttpRequestWrapper request) {
595         try {
596             responseCache.flushInvalidatedCacheEntriesFor(target, request);
597         } catch (final IOException ioe) {
598             log.warn("Unable to flush invalidated entries from cache", ioe);
599         }
600     }
601 
602     private HttpResponse generateCachedResponse(final HttpRequestWrapper request,
603             final HttpContext context, final HttpCacheEntry entry, final Date now) {
604         final HttpResponse cachedResponse;
605         if (request.containsHeader(HeaderConstants.IF_NONE_MATCH)
606                 || request.containsHeader(HeaderConstants.IF_MODIFIED_SINCE)) {
607             cachedResponse = responseGenerator.generateNotModifiedResponse(entry);
608         } else {
609             cachedResponse = responseGenerator.generateResponse(request, entry);
610         }
611         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
612         if (validityPolicy.getStalenessSecs(entry, now) > 0L) {
613             cachedResponse.addHeader(HeaderConstants.WARNING,"110 localhost \"Response is stale\"");
614         }
615         return cachedResponse;
616     }
617 
618     private HttpResponse handleRevalidationFailure(final HttpRequestWrapper request,
619             final HttpContext context, final HttpCacheEntry entry, final Date now) {
620         if (staleResponseNotAllowed(request, entry, now)) {
621             return generateGatewayTimeout(context);
622         } else {
623             return unvalidatedCacheHit(request, context, entry);
624         }
625     }
626 
627     private HttpResponse generateGatewayTimeout(final HttpContext context) {
628         setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
629         return new BasicHttpResponse(HttpVersion.HTTP_1_1,
630                 HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout");
631     }
632 
633     private HttpResponse unvalidatedCacheHit(
634             final HttpRequestWrapper request,
635             final HttpContext context,
636             final HttpCacheEntry entry) {
637         final HttpResponse cachedResponse = responseGenerator.generateResponse(request, entry);
638         setResponseStatus(context, CacheResponseStatus.CACHE_HIT);
639         cachedResponse.addHeader(HeaderConstants.WARNING, "111 localhost \"Revalidation failed\"");
640         return cachedResponse;
641     }
642 
643     private boolean staleResponseNotAllowed(final HttpRequestWrapper request,
644             final HttpCacheEntry entry, final Date now) {
645         return validityPolicy.mustRevalidate(entry)
646             || (isSharedCache() && validityPolicy.proxyRevalidate(entry))
647             || explicitFreshnessRequest(request, entry, now);
648     }
649 
650     private boolean mayCallBackend(final HttpRequestWrapper request) {
651         for (final Header h: request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
652             for (final HeaderElement elt : h.getElements()) {
653                 if ("only-if-cached".equals(elt.getName())) {
654                     log.trace("Request marked only-if-cached");
655                     return false;
656                 }
657             }
658         }
659         return true;
660     }
661 
662     private boolean explicitFreshnessRequest(final HttpRequestWrapper request, final HttpCacheEntry entry, final Date now) {
663         for(final Header h : request.getHeaders(HeaderConstants.CACHE_CONTROL)) {
664             for(final HeaderElement elt : h.getElements()) {
665                 if (HeaderConstants.CACHE_CONTROL_MAX_STALE.equals(elt.getName())) {
666                     try {
667                         final int maxstale = Integer.parseInt(elt.getValue());
668                         final long age = validityPolicy.getCurrentAgeSecs(entry, now);
669                         final long lifetime = validityPolicy.getFreshnessLifetimeSecs(entry);
670                         if (age - lifetime > maxstale) {
671                             return true;
672                         }
673                     } catch (final NumberFormatException nfe) {
674                         return true;
675                     }
676                 } else if (HeaderConstants.CACHE_CONTROL_MIN_FRESH.equals(elt.getName())
677                             || HeaderConstants.CACHE_CONTROL_MAX_AGE.equals(elt.getName())) {
678                     return true;
679                 }
680             }
681         }
682         return false;
683     }
684 
685     private String generateViaHeader(final HttpMessage msg) {
686 
687         final ProtocolVersion pv = msg.getProtocolVersion();
688         final String existingEntry = viaHeaders.get(pv);
689         if (existingEntry != null) {
690             return existingEntry;
691         }
692 
693         final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
694         final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
695 
696         final String value;
697         if ("http".equalsIgnoreCase(pv.getProtocol())) {
698             value = String.format("%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getMajor(), pv.getMinor(),
699                     release);
700         } else {
701             value = String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))", pv.getProtocol(), pv.getMajor(),
702                     pv.getMinor(), release);
703         }
704         viaHeaders.put(pv, value);
705 
706         return value;
707     }
708 
709     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
710         if (context != null) {
711             context.setAttribute(CACHE_RESPONSE_STATUS, value);
712         }
713     }
714 
715     /**
716      * Reports whether this {@code CachingHttpClient} implementation
717      * supports byte-range requests as specified by the {@code Range}
718      * and {@code Content-Range} headers.
719      * @return {@code true} if byte-range requests are supported
720      */
721     public boolean supportsRangeAndContentRangeHeaders() {
722         return SUPPORTS_RANGE_AND_CONTENT_RANGE_HEADERS;
723     }
724 
725     /**
726      * Reports whether this {@code CachingHttpClient} is configured as
727      * a shared (public) or non-shared (private) cache. See {@link
728      * CacheConfig#setSharedCache(boolean)}.
729      * @return {@code true} if we are behaving as a shared (public)
730      *   cache
731      */
732     public boolean isSharedCache() {
733         return sharedCache;
734     }
735 
736     Date getCurrentDate() {
737         return new Date();
738     }
739 
740     boolean clientRequestsOurOptions(final HttpRequest request) {
741         final RequestLine line = request.getRequestLine();
742 
743         if (!HeaderConstants.OPTIONS_METHOD.equals(line.getMethod())) {
744             return false;
745         }
746 
747         if (!"*".equals(line.getUri())) {
748             return false;
749         }
750 
751         final Header h = request.getFirstHeader(HeaderConstants.MAX_FORWARDS);
752         if (!"0".equals(h != null ? h.getValue() : null)) {
753             return false;
754         }
755 
756         return true;
757     }
758 
759     HttpResponse callBackend(final HttpHost target, final HttpRequestWrapper request, final HttpContext context)
760             throws IOException {
761 
762         final Date requestDate = getCurrentDate();
763 
764         log.trace("Calling the backend");
765         final HttpResponse backendResponse = backend.execute(target, request, context);
766         backendResponse.addHeader("Via", generateViaHeader(backendResponse));
767         return handleBackendResponse(target, request, requestDate, getCurrentDate(),
768                 backendResponse);
769 
770     }
771 
772     private boolean revalidationResponseIsTooOld(final HttpResponse backendResponse,
773             final HttpCacheEntry cacheEntry) {
774         final Header entryDateHeader = cacheEntry.getFirstHeader(HTTP.DATE_HEADER);
775         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
776         if (entryDateHeader != null && responseDateHeader != null) {
777             try {
778                 final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
779                 final Date respDate = DateUtils.parseDate(responseDateHeader.getValue());
780                 if (respDate.before(entryDate)) {
781                     return true;
782                 }
783             } catch (final DateParseException e) {
784                 // either backend response or cached entry did not have a valid
785                 // Date header, so we can't tell if they are out of order
786                 // according to the origin clock; thus we can skip the
787                 // unconditional retry recommended in 13.2.6 of RFC 2616.
788             }
789         }
790         return false;
791     }
792 
793     HttpResponse negotiateResponseFromVariants(final HttpHost target,
794             final HttpRequestWrapper request, final HttpContext context,
795             final Map<String, Variant> variants) throws IOException {
796         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder
797             .buildConditionalRequestFromVariants(request, variants);
798 
799         final Date requestDate = getCurrentDate();
800         final HttpResponse backendResponse = backend.execute(target, conditionalRequest, context);
801         final Date responseDate = getCurrentDate();
802 
803         backendResponse.addHeader("Via", generateViaHeader(backendResponse));
804 
805         if (backendResponse.getStatusLine().getStatusCode() != HttpStatus.SC_NOT_MODIFIED) {
806             return handleBackendResponse(target, request, requestDate, responseDate, backendResponse);
807         }
808 
809         final Header resultEtagHeader = backendResponse.getFirstHeader(HeaderConstants.ETAG);
810         if (resultEtagHeader == null) {
811             log.warn("304 response did not contain ETag");
812             return callBackend(target, request, context);
813         }
814 
815         final String resultEtag = resultEtagHeader.getValue();
816         final Variant matchingVariant = variants.get(resultEtag);
817         if (matchingVariant == null) {
818             log.debug("304 response did not contain ETag matching one sent in If-None-Match");
819             return callBackend(target, request, context);
820         }
821 
822         final HttpCacheEntry matchedEntry = matchingVariant.getEntry();
823 
824         if (revalidationResponseIsTooOld(backendResponse, matchedEntry)) {
825             IOUtils.consume(backendResponse.getEntity());
826             return retryRequestUnconditionally(target, request, context,
827                     matchedEntry);
828         }
829 
830         recordCacheUpdate(context);
831 
832         final HttpCacheEntry responseEntry = getUpdatedVariantEntry(target,
833                 conditionalRequest, requestDate, responseDate, backendResponse,
834                 matchingVariant, matchedEntry);
835 
836         final HttpResponse resp = responseGenerator.generateResponse(request, responseEntry);
837         tryToUpdateVariantMap(target, request, matchingVariant);
838 
839         if (shouldSendNotModifiedResponse(request, responseEntry)) {
840             return responseGenerator.generateNotModifiedResponse(responseEntry);
841         }
842 
843         return resp;
844     }
845 
846     private HttpResponse retryRequestUnconditionally(final HttpHost target,
847             final HttpRequestWrapper request, final HttpContext context,
848             final HttpCacheEntry matchedEntry) throws IOException {
849         final HttpRequestWrapper unconditional = conditionalRequestBuilder
850             .buildUnconditionalRequest(request, matchedEntry);
851         return callBackend(target, unconditional, context);
852     }
853 
854     private HttpCacheEntry getUpdatedVariantEntry(final HttpHost target,
855             final HttpRequestWrapper conditionalRequest, final Date requestDate,
856             final Date responseDate, final HttpResponse backendResponse,
857             final Variant matchingVariant, final HttpCacheEntry matchedEntry) {
858         HttpCacheEntry responseEntry = matchedEntry;
859         try {
860             responseEntry = responseCache.updateVariantCacheEntry(target, conditionalRequest,
861                     matchedEntry, backendResponse, requestDate, responseDate, matchingVariant.getCacheKey());
862         } catch (final IOException ioe) {
863             log.warn("Could not update cache entry", ioe);
864         }
865         return responseEntry;
866     }
867 
868     private void tryToUpdateVariantMap(final HttpHost target, final HttpRequestWrapper request,
869             final Variant matchingVariant) {
870         try {
871             responseCache.reuseVariantEntryFor(target, request, matchingVariant);
872         } catch (final IOException ioe) {
873             log.warn("Could not update cache entry to reuse variant", ioe);
874         }
875     }
876 
877     private boolean shouldSendNotModifiedResponse(final HttpRequestWrapper request,
878             final HttpCacheEntry responseEntry) {
879         return (suitabilityChecker.isConditional(request)
880                 && suitabilityChecker.allConditionalsMatch(request, responseEntry, new Date()));
881     }
882 
883     HttpResponse revalidateCacheEntry(
884             final HttpHost target,
885             final HttpRequestWrapper request,
886             final HttpContext context,
887             final HttpCacheEntry cacheEntry) throws IOException, ProtocolException {
888 
889         final HttpRequestWrapper conditionalRequest = conditionalRequestBuilder.buildConditionalRequest(request, cacheEntry);
890 
891         Date requestDate = getCurrentDate();
892         HttpResponse backendResponse = backend.execute(target, conditionalRequest, context);
893         Date responseDate = getCurrentDate();
894 
895         if (revalidationResponseIsTooOld(backendResponse, cacheEntry)) {
896             IOUtils.consume(backendResponse.getEntity());
897             final HttpRequest unconditional = conditionalRequestBuilder
898                 .buildUnconditionalRequest(request, cacheEntry);
899             requestDate = getCurrentDate();
900             backendResponse = backend.execute(target, unconditional, context);
901             responseDate = getCurrentDate();
902         }
903 
904         backendResponse.addHeader(HeaderConstants.VIA, generateViaHeader(backendResponse));
905 
906         final int statusCode = backendResponse.getStatusLine().getStatusCode();
907         if (statusCode == HttpStatus.SC_NOT_MODIFIED || statusCode == HttpStatus.SC_OK) {
908             recordCacheUpdate(context);
909         }
910 
911         if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
912             final HttpCacheEntry updatedEntry = responseCache.updateCacheEntry(target, request, cacheEntry,
913                     backendResponse, requestDate, responseDate);
914             if (suitabilityChecker.isConditional(request)
915                     && suitabilityChecker.allConditionalsMatch(request, updatedEntry, new Date())) {
916                 return responseGenerator.generateNotModifiedResponse(updatedEntry);
917             }
918             return responseGenerator.generateResponse(request, updatedEntry);
919         }
920 
921         if (staleIfErrorAppliesTo(statusCode)
922             && !staleResponseNotAllowed(request, cacheEntry, getCurrentDate())
923             && validityPolicy.mayReturnStaleIfError(request, cacheEntry, responseDate)) {
924             final HttpResponse cachedResponse = responseGenerator.generateResponse(request, cacheEntry);
925             cachedResponse.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\"");
926             final HttpEntity errorBody = backendResponse.getEntity();
927             if (errorBody != null) {
928                 IOUtils.consume(errorBody);
929             }
930             return cachedResponse;
931         }
932 
933         return handleBackendResponse(target, conditionalRequest, requestDate, responseDate,
934                                      backendResponse);
935     }
936 
937     private boolean staleIfErrorAppliesTo(final int statusCode) {
938         return statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
939                 || statusCode == HttpStatus.SC_BAD_GATEWAY
940                 || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
941                 || statusCode == HttpStatus.SC_GATEWAY_TIMEOUT;
942     }
943 
944     HttpResponse handleBackendResponse(
945             final HttpHost target,
946             final HttpRequestWrapper request,
947             final Date requestDate,
948             final Date responseDate,
949             final HttpResponse backendResponse) throws IOException {
950 
951         log.trace("Handling Backend response");
952         responseCompliance.ensureProtocolCompliance(request, backendResponse);
953 
954         final boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse);
955         responseCache.flushInvalidatedCacheEntriesFor(target, request, backendResponse);
956         if (cacheable &&
957             !alreadyHaveNewerCacheEntry(target, request, backendResponse)) {
958             try {
959                 storeRequestIfModifiedSinceFor304Response(request, backendResponse);
960                 return responseCache.cacheAndReturnResponse(target, request, backendResponse, requestDate,
961                         responseDate);
962             } catch (final IOException ioe) {
963                 log.warn("Unable to store entries in cache", ioe);
964             }
965         }
966         if (!cacheable) {
967             try {
968                 responseCache.flushCacheEntriesFor(target, request);
969             } catch (final IOException ioe) {
970                 log.warn("Unable to flush invalid cache entries", ioe);
971             }
972         }
973         return backendResponse;
974     }
975 
976     /**
977      * For 304 Not modified responses, adds a "Last-Modified" header with the
978      * value of the "If-Modified-Since" header passed in the request. This
979      * header is required to be able to reuse match the cache entry for
980      * subsequent requests but as defined in http specifications it is not
981      * included in 304 responses by backend servers. This header will not be
982      * included in the resulting response.
983      */
984     private void storeRequestIfModifiedSinceFor304Response(
985             final HttpRequest request, final HttpResponse backendResponse) {
986         if (backendResponse.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
987             final Header h = request.getFirstHeader("If-Modified-Since");
988             if (h != null) {
989                 backendResponse.addHeader("Last-Modified", h.getValue());
990             }
991         }
992     }
993 
994     private boolean alreadyHaveNewerCacheEntry(final HttpHost target, final HttpRequest request,
995             final HttpResponse backendResponse) {
996         HttpCacheEntry existing = null;
997         try {
998             existing = responseCache.getCacheEntry(target, request);
999         } catch (final IOException ioe) {
1000             // nop
1001         }
1002         if (existing == null) {
1003             return false;
1004         }
1005         final Header entryDateHeader = existing.getFirstHeader(HTTP.DATE_HEADER);
1006         if (entryDateHeader == null) {
1007             return false;
1008         }
1009         final Header responseDateHeader = backendResponse.getFirstHeader(HTTP.DATE_HEADER);
1010         if (responseDateHeader == null) {
1011             return false;
1012         }
1013         try {
1014             final Date entryDate = DateUtils.parseDate(entryDateHeader.getValue());
1015             final Date responseDate = DateUtils.parseDate(responseDateHeader.getValue());
1016             return responseDate.before(entryDate);
1017         } catch (final DateParseException e) {
1018             // Empty on Purpose
1019         }
1020         return false;
1021     }
1022 
1023     static class AsynchronousValidator {
1024         private final CachingHttpClient cachingClient;
1025         private final ExecutorService executor;
1026         private final Set<String> queued;
1027         private final CacheKeyGenerator cacheKeyGenerator;
1028 
1029         private final Log log = LogFactory.getLog(getClass());
1030 
1031         /**
1032          * Create AsynchronousValidator which will make revalidation requests
1033          * using the supplied {@link CachingHttpClient}, and
1034          * a {@link ThreadPoolExecutor} generated according to the thread
1035          * pool settings provided in the given {@link CacheConfig}.
1036          * @param cachingClient used to execute asynchronous requests
1037          * @param config specifies thread pool settings. See
1038          * {@link CacheConfig#getAsynchronousWorkersMax()},
1039          * {@link CacheConfig#getAsynchronousWorkersCore()},
1040          * {@link CacheConfig#getAsynchronousWorkerIdleLifetimeSecs()},
1041          * and {@link CacheConfig#getRevalidationQueueSize()}.
1042          */
1043         public AsynchronousValidator(final CachingHttpClient cachingClient,
1044                 final CacheConfig config) {
1045             this(cachingClient,
1046                     new ThreadPoolExecutor(config.getAsynchronousWorkersCore(),
1047                             config.getAsynchronousWorkersMax(),
1048                             config.getAsynchronousWorkerIdleLifetimeSecs(),
1049                             TimeUnit.SECONDS,
1050                             new ArrayBlockingQueue<Runnable>(config.getRevalidationQueueSize()))
1051                     );
1052         }
1053 
1054         /**
1055          * Create AsynchronousValidator which will make revalidation requests
1056          * using the supplied {@link CachingHttpClient} and
1057          * {@link ExecutorService}.
1058          * @param cachingClient used to execute asynchronous requests
1059          * @param executor used to manage a thread pool of revalidation workers
1060          */
1061         AsynchronousValidator(final CachingHttpClient cachingClient,
1062                 final ExecutorService executor) {
1063             this.cachingClient = cachingClient;
1064             this.executor = executor;
1065             this.queued = new HashSet<String>();
1066             this.cacheKeyGenerator = new CacheKeyGenerator();
1067         }
1068 
1069         /**
1070          * Schedules an asynchronous revalidation
1071          *
1072          * @param target
1073          * @param request
1074          * @param context
1075          * @param entry
1076          */
1077         public synchronized void revalidateCacheEntry(final HttpHost target,
1078                 final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry) {
1079             // getVariantURI will fall back on getURI if no variants exist
1080             final String uri = cacheKeyGenerator.getVariantURI(target, request, entry);
1081 
1082             if (!queued.contains(uri)) {
1083                 final AsynchronousValidationRequest revalidationRequest =
1084                     new AsynchronousValidationRequest(this, cachingClient, target,
1085                             request, context, entry, uri);
1086 
1087                 try {
1088                     executor.execute(revalidationRequest);
1089                     queued.add(uri);
1090                 } catch (final RejectedExecutionException ree) {
1091                     log.debug("Revalidation for [" + uri + "] not scheduled: " + ree);
1092                 }
1093             }
1094         }
1095 
1096         /**
1097          * Removes an identifier from the internal list of revalidation jobs in
1098          * progress.  This is meant to be called by
1099          * {@link AsynchronousValidationRequest#run()} once the revalidation is
1100          * complete, using the identifier passed in during constructions.
1101          * @param identifier
1102          */
1103         synchronized void markComplete(final String identifier) {
1104             queued.remove(identifier);
1105         }
1106 
1107         Set<String> getScheduledIdentifiers() {
1108             return Collections.unmodifiableSet(queued);
1109         }
1110 
1111         ExecutorService getExecutor() {
1112             return executor;
1113         }
1114     }
1115 
1116     static class AsynchronousValidationRequest implements Runnable {
1117         private final AsynchronousValidator parent;
1118         private final CachingHttpClient cachingClient;
1119         private final HttpHost target;
1120         private final HttpRequestWrapper request;
1121         private final HttpContext context;
1122         private final HttpCacheEntry cacheEntry;
1123         private final String identifier;
1124 
1125         private final Log log = LogFactory.getLog(getClass());
1126 
1127         /**
1128          * Used internally by {@link AsynchronousValidator} to schedule a
1129          * revalidation.
1130          * @param cachingClient
1131          * @param target
1132          * @param request
1133          * @param context
1134          * @param cacheEntry
1135          * @param bookKeeping
1136          * @param identifier
1137          */
1138         AsynchronousValidationRequest(final AsynchronousValidator parent,
1139                 final CachingHttpClient cachingClient, final HttpHost target,
1140                 final HttpRequestWrapper request, final HttpContext context,
1141                 final HttpCacheEntry cacheEntry,
1142                 final String identifier) {
1143             this.parent = parent;
1144             this.cachingClient = cachingClient;
1145             this.target = target;
1146             this.request = request;
1147             this.context = context;
1148             this.cacheEntry = cacheEntry;
1149             this.identifier = identifier;
1150         }
1151 
1152         @Override
1153         public void run() {
1154             try {
1155                 cachingClient.revalidateCacheEntry(target, request, context, cacheEntry);
1156             } catch (final IOException ioe) {
1157                 log.debug("Asynchronous revalidation failed due to exception: " + ioe);
1158             } catch (final ProtocolException pe) {
1159                 log.error("ProtocolException thrown during asynchronous revalidation: " + pe);
1160             } finally {
1161                 parent.markComplete(identifier);
1162             }
1163         }
1164 
1165         String getIdentifier() {
1166             return identifier;
1167         }
1168 
1169     }
1170 
1171 }