View Javadoc

1   //
2   //  ========================================================================
3   //  Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
4   //  ------------------------------------------------------------------------
5   //  All rights reserved. This program and the accompanying materials
6   //  are made available under the terms of the Eclipse Public License v1.0
7   //  and Apache License v2.0 which accompanies this distribution.
8   //
9   //      The Eclipse Public License is available at
10  //      http://www.eclipse.org/legal/epl-v10.html
11  //
12  //      The Apache License v2.0 is available at
13  //      http://www.opensource.org/licenses/apache2.0.php
14  //
15  //  You may elect to redistribute this code under either of these licenses.
16  //  ========================================================================
17  //
18  
19  package org.eclipse.jetty.servlets;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.OutputStream;
24  import java.util.HashSet;
25  import java.util.Locale;
26  import java.util.Set;
27  import java.util.StringTokenizer;
28  import java.util.regex.Pattern;
29  import java.util.zip.Deflater;
30  
31  import javax.servlet.AsyncEvent;
32  import javax.servlet.AsyncListener;
33  import javax.servlet.FilterChain;
34  import javax.servlet.FilterConfig;
35  import javax.servlet.ServletContext;
36  import javax.servlet.ServletException;
37  import javax.servlet.ServletRequest;
38  import javax.servlet.ServletResponse;
39  import javax.servlet.http.HttpServletRequest;
40  import javax.servlet.http.HttpServletResponse;
41  
42  import org.eclipse.jetty.http.HttpMethod;
43  import org.eclipse.jetty.http.MimeTypes;
44  import org.eclipse.jetty.servlets.gzip.AbstractCompressedStream;
45  import org.eclipse.jetty.servlets.gzip.CompressedResponseWrapper;
46  import org.eclipse.jetty.servlets.gzip.DeflatedOutputStream;
47  import org.eclipse.jetty.servlets.gzip.GzipOutputStream;
48  import org.eclipse.jetty.util.URIUtil;
49  import org.eclipse.jetty.util.log.Log;
50  import org.eclipse.jetty.util.log.Logger;
51  
52  /* ------------------------------------------------------------ */
53  /** GZIP Filter
54   * This filter will gzip or deflate the content of a response if: <ul>
55   * <li>The filter is mapped to a matching path</li>
56   * <li>accept-encoding header is set to either gzip, deflate or a combination of those</li>
57   * <li>The response status code is >=200 and <300
58   * <li>The content length is unknown or more than the <code>minGzipSize</code> initParameter or the minGzipSize is 0(default)</li>
59   * <li>If a list of mimeTypes is set by the <code>mimeTypes</code> init parameter, then the Content-Type is in the list.</li>
60   * <li>If no mimeType list is set, then the content-type is not in the list defined by <code>excludedMimeTypes</code></li>
61   * <li>No content-encoding is specified by the resource</li>
62   * </ul>
63   *
64   * <p>
65   * If both gzip and deflate are specified in the accept-encoding header, then gzip will be used.
66   * </p>
67   * <p>
68   * Compressing the content can greatly improve the network bandwidth usage, but at a cost of memory and
69   * CPU cycles. If this filter is mapped for static content, then use of efficient direct NIO may be
70   * prevented, thus use of the gzip mechanism of the {@link org.eclipse.jetty.servlet.DefaultServlet} is
71   * advised instead.
72   * </p>
73   * <p>
74   * This filter extends {@link UserAgentFilter} and if the the initParameter <code>excludedAgents</code>
75   * is set to a comma separated list of user agents, then these agents will be excluded from gzip content.
76   * </p>
77   * <p>Init Parameters:</p>
78   * <dl>
79   * <dt>bufferSize</dt>       <dd>The output buffer size. Defaults to 8192. Be careful as values <= 0 will lead to an
80   *                            {@link IllegalArgumentException}.
81   *                            See: {@link java.util.zip.GZIPOutputStream#GZIPOutputStream(java.io.OutputStream, int)}
82   *                            and: {@link java.util.zip.DeflaterOutputStream#DeflaterOutputStream(java.io.OutputStream, Deflater, int)}
83   * </dd>
84   * <dt>minGzipSize</dt>       <dd>Content will only be compressed if content length is either unknown or greater
85   *                            than <code>minGzipSize</code>.
86   * </dd>
87   * <dt>deflateCompressionLevel</dt>       <dd>The compression level used for deflate compression. (0-9).
88   *                            See: {@link java.util.zip.Deflater#Deflater(int, boolean)}
89   * </dd>
90   * <dt>deflateNoWrap</dt>       <dd>The noWrap setting for deflate compression. Defaults to true. (true/false)
91   *                            See: {@link java.util.zip.Deflater#Deflater(int, boolean)}
92   * </dd>
93   * <dt>methods</dt>       <dd>Comma separated list of HTTP methods to compress. If not set, only GET requests are compressed.
94   *  </dd>
95   * <dt>mimeTypes</dt>       <dd>Comma separated list of mime types to compress. If it is not set, then the excludedMimeTypes list is used.
96   * </dd>
97   * <dt>excludedMimeTypes</dt>       <dd>Comma separated list of mime types to never compress. If not set, then the default is the commonly known
98   * image, video, audio and compressed types.
99   * </dd>
100 
101  * <dt>excludedAgents</dt>       <dd>Comma separated list of user agents to exclude from compression. Does a
102  *                            {@link String#contains(CharSequence)} to check if the excluded agent occurs
103  *                            in the user-agent header. If it does -> no compression
104  * </dd>
105  * <dt>excludeAgentPatterns</dt>       <dd>Same as excludedAgents, but accepts regex patterns for more complex matching.
106  * </dd>
107  * <dt>excludePaths</dt>       <dd>Comma separated list of paths to exclude from compression.
108  *                            Does a {@link String#startsWith(String)} comparison to check if the path matches.
109  *                            If it does match -> no compression. To match subpaths use <code>excludePathPatterns</code>
110  *                            instead.
111  * </dd>
112  * <dt>excludePathPatterns</dt>       <dd>Same as excludePath, but accepts regex patterns for more complex matching.
113  * </dd>
114  * <dt>vary</dt>       <dd>Set to the value of the Vary header sent with responses that could be compressed.  By default it is 
115  *                            set to 'Vary: Accept-Encoding, User-Agent' since IE6 is excluded by default from the excludedAgents. 
116  *                            If user-agents are not to be excluded, then this can be set to 'Vary: Accept-Encoding'.  Note also 
117  *                            that shared caches may cache copies of a resource that is varied by User-Agent - one per variation of 
118  *                            the User-Agent, unless the cache does some normalization of the UA string.
119  * </dd>                         
120  * <dt>checkGzExists</dt>       <dd>If set to true, the filter check if a static resource with ".gz" appended exists.  If so then
121  *                            the normal processing is done so that the default servlet can send  the pre existing gz content.
122  *  </dd>
123  *  </dl>
124  */
125 public class GzipFilter extends UserAgentFilter
126 {
127     private static final Logger LOG = Log.getLogger(GzipFilter.class);
128     public final static String GZIP="gzip";
129     public final static String ETAG_GZIP="--gzip\"";
130     public final static String DEFLATE="deflate";
131     public final static String ETAG_DEFLATE="--deflate\"";
132     public final static String ETAG="o.e.j.s.GzipFilter.ETag";
133 
134     protected ServletContext _context;
135     protected final Set<String> _mimeTypes=new HashSet<>();
136     protected boolean _excludeMimeTypes;
137     protected int _bufferSize=8192;
138     protected int _minGzipSize=256;
139     protected int _deflateCompressionLevel=Deflater.DEFAULT_COMPRESSION;
140     protected boolean _deflateNoWrap = true;
141     protected boolean _checkGzExists = true;
142     
143     // non-static, as other GzipFilter instances may have different configurations
144     protected final ThreadLocal<Deflater> _deflater = new ThreadLocal<Deflater>();
145 
146     protected final static ThreadLocal<byte[]> _buffer= new ThreadLocal<byte[]>();
147 
148     protected final Set<String> _methods=new HashSet<String>();
149     protected Set<String> _excludedAgents;
150     protected Set<Pattern> _excludedAgentPatterns;
151     protected Set<String> _excludedPaths;
152     protected Set<Pattern> _excludedPathPatterns;
153     protected String _vary="Accept-Encoding, User-Agent";
154     
155     private static final int STATE_SEPARATOR = 0;
156     private static final int STATE_Q = 1;
157     private static final int STATE_QVALUE = 2;
158     private static final int STATE_DEFAULT = 3;
159 
160 
161     /* ------------------------------------------------------------ */
162     /**
163      * @see org.eclipse.jetty.servlets.UserAgentFilter#init(javax.servlet.FilterConfig)
164      */
165     @Override
166     public void init(FilterConfig filterConfig) throws ServletException
167     {
168         super.init(filterConfig);
169 
170         _context=filterConfig.getServletContext();
171         
172         String tmp=filterConfig.getInitParameter("bufferSize");
173         if (tmp!=null)
174             _bufferSize=Integer.parseInt(tmp);
175 
176         tmp=filterConfig.getInitParameter("minGzipSize");
177         if (tmp!=null)
178             _minGzipSize=Integer.parseInt(tmp);
179 
180         tmp=filterConfig.getInitParameter("deflateCompressionLevel");
181         if (tmp!=null)
182             _deflateCompressionLevel=Integer.parseInt(tmp);
183 
184         tmp=filterConfig.getInitParameter("deflateNoWrap");
185         if (tmp!=null)
186             _deflateNoWrap=Boolean.parseBoolean(tmp);
187 
188         tmp=filterConfig.getInitParameter("checkGzExists");
189         if (tmp!=null)
190             _checkGzExists=Boolean.parseBoolean(tmp);
191         
192         tmp=filterConfig.getInitParameter("methods");
193         if (tmp!=null)
194         {
195             StringTokenizer tok = new StringTokenizer(tmp,",",false);
196             while (tok.hasMoreTokens())
197                 _methods.add(tok.nextToken().trim().toUpperCase(Locale.ENGLISH));
198         }
199         else
200             _methods.add(HttpMethod.GET.asString());
201         
202         tmp=filterConfig.getInitParameter("mimeTypes");
203         if (tmp==null)
204         {
205             _excludeMimeTypes=true;
206             tmp=filterConfig.getInitParameter("excludedMimeTypes");
207             if (tmp==null)
208             {
209                 for (String type:MimeTypes.getKnownMimeTypes())
210                 {
211                     if (type.startsWith("image/")||
212                         type.startsWith("audio/")||
213                         type.startsWith("video/"))
214                         _mimeTypes.add(type);
215                     _mimeTypes.add("application/compress");
216                     _mimeTypes.add("application/zip");
217                     _mimeTypes.add("application/gzip");
218                 }
219             }
220             else
221             {
222                 StringTokenizer tok = new StringTokenizer(tmp,",",false);
223                 while (tok.hasMoreTokens())
224                     _mimeTypes.add(tok.nextToken().trim());
225             }
226         }
227         else
228         {
229             StringTokenizer tok = new StringTokenizer(tmp,",",false);
230             while (tok.hasMoreTokens())
231                 _mimeTypes.add(tok.nextToken().trim());
232         }
233         tmp=filterConfig.getInitParameter("excludedAgents");
234         if (tmp!=null)
235         {
236             _excludedAgents=new HashSet<String>();
237             StringTokenizer tok = new StringTokenizer(tmp,",",false);
238             while (tok.hasMoreTokens())
239                _excludedAgents.add(tok.nextToken().trim());
240         }
241 
242         tmp=filterConfig.getInitParameter("excludeAgentPatterns");
243         if (tmp!=null)
244         {
245             _excludedAgentPatterns=new HashSet<Pattern>();
246             StringTokenizer tok = new StringTokenizer(tmp,",",false);
247             while (tok.hasMoreTokens())
248                 _excludedAgentPatterns.add(Pattern.compile(tok.nextToken().trim()));
249         }
250 
251         tmp=filterConfig.getInitParameter("excludePaths");
252         if (tmp!=null)
253         {
254             _excludedPaths=new HashSet<String>();
255             StringTokenizer tok = new StringTokenizer(tmp,",",false);
256             while (tok.hasMoreTokens())
257                 _excludedPaths.add(tok.nextToken().trim());
258         }
259 
260         tmp=filterConfig.getInitParameter("excludePathPatterns");
261         if (tmp!=null)
262         {
263             _excludedPathPatterns=new HashSet<Pattern>();
264             StringTokenizer tok = new StringTokenizer(tmp,",",false);
265             while (tok.hasMoreTokens())
266                 _excludedPathPatterns.add(Pattern.compile(tok.nextToken().trim()));
267         }
268         
269         tmp=filterConfig.getInitParameter("vary");
270         if (tmp!=null)
271             _vary=tmp;
272     }
273 
274     /* ------------------------------------------------------------ */
275     /**
276      * @see org.eclipse.jetty.servlets.UserAgentFilter#destroy()
277      */
278     @Override
279     public void destroy()
280     {
281     }
282 
283     /* ------------------------------------------------------------ */
284     /**
285      * @see org.eclipse.jetty.servlets.UserAgentFilter#doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain)
286      */
287     @Override
288     public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain)
289         throws IOException, ServletException
290     {
291         HttpServletRequest request=(HttpServletRequest)req;
292         HttpServletResponse response=(HttpServletResponse)res;
293 
294         // If not a supported method or it is an Excluded URI - no Vary because no matter what client, this URI is always excluded
295         String requestURI = request.getRequestURI();
296         if (!_methods.contains(request.getMethod()) || isExcludedPath(requestURI))
297         {
298             super.doFilter(request,response,chain);
299             return;
300         }
301         
302         // Exclude non compressible mime-types known from URI extension. - no Vary because no matter what client, this URI is always excluded
303         if (_mimeTypes.size()>0)
304         {
305             String mimeType = _context.getMimeType(request.getRequestURI());
306             
307             if (mimeType!=null && _mimeTypes.contains(mimeType)==_excludeMimeTypes)
308             {
309                 // handle normally without setting vary header
310                 super.doFilter(request,response,chain);
311                 return;
312             }
313         }
314 
315         if (_checkGzExists && request.getServletContext()!=null)
316         {
317             String path=request.getServletContext().getRealPath(URIUtil.addPaths(request.getServletPath(),request.getPathInfo()));
318             if (path!=null)
319             {
320                 File gz=new File(path+".gz");
321                 if (gz.exists())
322                 {
323                     // allow default servlet to handle
324                     super.doFilter(request,response,chain);
325                     return;
326                 }
327             }
328         }
329         
330         // Excluded User-Agents
331         String ua = getUserAgent(request);
332         boolean ua_excluded=ua!=null&&isExcludedAgent(ua);
333         
334         // Acceptable compression type
335         String compressionType = ua_excluded?null:selectCompression(request.getHeader("accept-encoding"));
336 
337         // Special handling for etags
338         String etag = request.getHeader("If-None-Match"); 
339         if (etag!=null)
340         {
341             int dd=etag.indexOf("--");
342             if (dd>0)
343                 request.setAttribute(ETAG,etag.substring(0,dd)+(etag.endsWith("\"")?"\"":""));
344         }
345 
346         CompressedResponseWrapper wrappedResponse = createWrappedResponse(request,response,compressionType);
347 
348         boolean exceptional=true;
349         try
350         {
351             super.doFilter(request,wrappedResponse,chain);
352             exceptional=false;
353         }
354         finally
355         {
356             if (request.isAsyncStarted())
357             {
358                 request.getAsyncContext().addListener(new FinishOnCompleteListener(wrappedResponse));
359             }
360             else if (exceptional && !response.isCommitted())
361             {
362                 wrappedResponse.resetBuffer();
363                 wrappedResponse.noCompression();
364             }
365             else
366                 wrappedResponse.finish();
367         }
368     }
369 
370     /* ------------------------------------------------------------ */
371     private String selectCompression(String encodingHeader)
372     {
373         // TODO, this could be a little more robust.
374         // prefer gzip over deflate
375         String compression = null;
376         if (encodingHeader!=null)
377         {
378             
379             String[] encodings = getEncodings(encodingHeader);
380             if (encodings != null)
381             {
382                 for (int i=0; i< encodings.length; i++)
383                 {
384                     if (encodings[i].toLowerCase(Locale.ENGLISH).contains(GZIP))
385                     {
386                         if (isEncodingAcceptable(encodings[i]))
387                         {
388                             compression = GZIP;
389                             break; //prefer Gzip over deflate
390                         }
391                     }
392 
393                     if (encodings[i].toLowerCase(Locale.ENGLISH).contains(DEFLATE))
394                     {
395                         if (isEncodingAcceptable(encodings[i]))
396                         {
397                             compression = DEFLATE; //Keep checking in case gzip is acceptable
398                         }
399                     }
400                 }
401             }
402         }
403         return compression;
404     }
405     
406     
407     private String[] getEncodings (String encodingHeader)
408     {
409         if (encodingHeader == null)
410             return null;
411         return encodingHeader.split(",");
412     }
413     
414     private boolean isEncodingAcceptable(String encoding)
415     {    
416         int state = STATE_DEFAULT;
417         int qvalueIdx = -1;
418         for (int i=0;i<encoding.length();i++)
419         {
420             char c = encoding.charAt(i);
421             switch (state)
422             {
423                 case STATE_DEFAULT:
424                 {
425                     if (';' == c)
426                         state = STATE_SEPARATOR;
427                     break;
428                 }
429                 case STATE_SEPARATOR:
430                 {
431                     if ('q' == c || 'Q' == c)
432                         state = STATE_Q;
433                     break;
434                 }
435                 case STATE_Q:
436                 {
437                     if ('=' == c)
438                         state = STATE_QVALUE;
439                     break;
440                 }
441                 case STATE_QVALUE:
442                 {
443                     if (qvalueIdx < 0 && '0' == c || '1' == c)
444                         qvalueIdx = i;
445                     break;
446                 }
447             }
448         }
449         
450         if (qvalueIdx < 0)
451             return true;
452                
453         if ("0".equals(encoding.substring(qvalueIdx).trim()))
454             return false;
455         return true;
456     }
457     
458 
459     protected CompressedResponseWrapper createWrappedResponse(HttpServletRequest request, HttpServletResponse response, final String compressionType)
460     {
461         CompressedResponseWrapper wrappedResponse = null;
462         wrappedResponse = new CompressedResponseWrapper(request,response)
463         {
464             @Override
465             protected AbstractCompressedStream newCompressedStream(HttpServletRequest request, HttpServletResponse response) throws IOException
466             {
467                 return new AbstractCompressedStream(compressionType,request,this,_vary)
468                 {
469                     private Deflater _allocatedDeflater;
470                     private byte[] _allocatedBuffer;
471 
472                     @Override
473                     protected OutputStream createStream() throws IOException
474                     {
475                         if (compressionType == null)
476                         {
477                             return null;
478                         }
479                         
480                         // acquire deflater instance
481                         _allocatedDeflater = _deflater.get();   
482                         if (_allocatedDeflater==null)
483                             _allocatedDeflater = new Deflater(_deflateCompressionLevel,_deflateNoWrap);
484                         else
485                         {
486                             _deflater.remove();
487                             _allocatedDeflater.reset();
488                         }
489                         
490                         // acquire buffer
491                         _allocatedBuffer = _buffer.get();
492                         if (_allocatedBuffer==null)
493                             _allocatedBuffer = new byte[_bufferSize];
494                         else
495                         {
496                             _buffer.remove();
497                         }
498                         
499                         switch (compressionType)
500                         {
501                             case GZIP:
502                                 return new GzipOutputStream(_response.getOutputStream(),_allocatedDeflater,_allocatedBuffer);
503                             case DEFLATE:
504                                 return new DeflatedOutputStream(_response.getOutputStream(),_allocatedDeflater,_allocatedBuffer);
505                         }
506                         throw new IllegalStateException(compressionType + " not supported");
507                     }
508 
509                     @Override
510                     public void finish() throws IOException
511                     {
512                         super.finish();
513                         if (_allocatedDeflater != null && _deflater.get() == null)
514                         {
515                             _deflater.set(_allocatedDeflater);
516                         }
517                         if (_allocatedBuffer != null && _buffer.get() == null)
518                         {
519                             _buffer.set(_allocatedBuffer);
520                         }
521                     }
522                 };
523             }
524         };
525         configureWrappedResponse(wrappedResponse);
526         return wrappedResponse;
527     }
528 
529     protected void configureWrappedResponse(CompressedResponseWrapper wrappedResponse)
530     {
531         wrappedResponse.setMimeTypes(_mimeTypes,_excludeMimeTypes);
532         wrappedResponse.setBufferSize(_bufferSize);
533         wrappedResponse.setMinCompressSize(_minGzipSize);
534     }
535 
536     private class FinishOnCompleteListener implements AsyncListener
537     {    
538         private CompressedResponseWrapper wrappedResponse;
539 
540         public FinishOnCompleteListener(CompressedResponseWrapper wrappedResponse)
541         {
542             this.wrappedResponse = wrappedResponse;
543         }
544 
545         @Override
546         public void onComplete(AsyncEvent event) throws IOException
547         {          
548             try
549             {
550                 wrappedResponse.finish();
551             }
552             catch (IOException e)
553             {
554                 LOG.warn(e);
555             }
556         }
557 
558         @Override
559         public void onTimeout(AsyncEvent event) throws IOException
560         {
561         }
562 
563         @Override
564         public void onError(AsyncEvent event) throws IOException
565         {
566         }
567 
568         @Override
569         public void onStartAsync(AsyncEvent event) throws IOException
570         {
571         }
572     }
573 
574     /**
575      * Checks to see if the userAgent is excluded
576      *
577      * @param ua
578      *            the user agent
579      * @return boolean true if excluded
580      */
581     private boolean isExcludedAgent(String ua)
582     {
583         if (ua == null)
584             return false;
585 
586         if (_excludedAgents != null)
587         {
588             if (_excludedAgents.contains(ua))
589             {
590                 return true;
591             }
592         }
593         if (_excludedAgentPatterns != null)
594         {
595             for (Pattern pattern : _excludedAgentPatterns)
596             {
597                 if (pattern.matcher(ua).matches())
598                 {
599                     return true;
600                 }
601             }
602         }
603 
604         return false;
605     }
606 
607     /**
608      * Checks to see if the path is excluded
609      *
610      * @param requestURI
611      *            the request uri
612      * @return boolean true if excluded
613      */
614     private boolean isExcludedPath(String requestURI)
615     {
616         if (requestURI == null)
617             return false;
618         if (_excludedPaths != null)
619         {
620             for (String excludedPath : _excludedPaths)
621             {
622                 if (requestURI.startsWith(excludedPath))
623                 {
624                     return true;
625                 }
626             }
627         }
628         if (_excludedPathPatterns != null)
629         {
630             for (Pattern pattern : _excludedPathPatterns)
631             {
632                 if (pattern.matcher(requestURI).matches())
633                 {
634                     return true;
635                 }
636             }
637         }
638         return false;
639     }
640 }