View Javadoc

1   //
2   //  ========================================================================
3   //  Copyright (c) 1995-2015 Mort Bay Consulting Pty. Ltd.
4   //  ------------------------------------------------------------------------
5   //  All rights reserved. This program and the accompanying materials
6   //  are made available under the terms of the Eclipse Public License v1.0
7   //  and Apache License v2.0 which accompanies this distribution.
8   //
9   //      The Eclipse Public License is available at
10  //      http://www.eclipse.org/legal/epl-v10.html
11  //
12  //      The Apache License v2.0 is available at
13  //      http://www.opensource.org/licenses/apache2.0.php
14  //
15  //  You may elect to redistribute this code under either of these licenses.
16  //  ========================================================================
17  //
18  
19  package org.eclipse.jetty.servlets;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.OutputStream;
24  import java.util.HashSet;
25  import java.util.Locale;
26  import java.util.Set;
27  import java.util.StringTokenizer;
28  import java.util.regex.Pattern;
29  import java.util.zip.Deflater;
30  
31  import javax.servlet.AsyncEvent;
32  import javax.servlet.AsyncListener;
33  import javax.servlet.FilterChain;
34  import javax.servlet.FilterConfig;
35  import javax.servlet.ServletContext;
36  import javax.servlet.ServletException;
37  import javax.servlet.ServletRequest;
38  import javax.servlet.ServletResponse;
39  import javax.servlet.http.HttpServletRequest;
40  import javax.servlet.http.HttpServletResponse;
41  
42  import org.eclipse.jetty.http.HttpMethod;
43  import org.eclipse.jetty.http.MimeTypes;
44  import org.eclipse.jetty.servlets.gzip.AbstractCompressedStream;
45  import org.eclipse.jetty.servlets.gzip.CompressedResponseWrapper;
46  import org.eclipse.jetty.servlets.gzip.DeflatedOutputStream;
47  import org.eclipse.jetty.servlets.gzip.GzipOutputStream;
48  import org.eclipse.jetty.util.URIUtil;
49  import org.eclipse.jetty.util.log.Log;
50  import org.eclipse.jetty.util.log.Logger;
51  
52  /* ------------------------------------------------------------ */
53  /** GZIP Filter
54   * This filter will gzip or deflate the content of a response if: <ul>
55   * <li>The filter is mapped to a matching path</li>
56   * <li>accept-encoding header is set to either gzip, deflate or a combination of those</li>
57   * <li>The response status code is >=200 and <300
58   * <li>The content length is unknown or more than the <code>minGzipSize</code> initParameter or the minGzipSize is 0(default)</li>
59   * <li>If a list of mimeTypes is set by the <code>mimeTypes</code> init parameter, then the Content-Type is in the list.</li>
60   * <li>If no mimeType list is set, then the content-type is not in the list defined by <code>excludedMimeTypes</code></li>
61   * <li>No content-encoding is specified by the resource</li>
62   * </ul>
63   *
64   * <p>
65   * If both gzip and deflate are specified in the accept-encoding header, then gzip will be used.
66   * </p>
67   * <p>
68   * Compressing the content can greatly improve the network bandwidth usage, but at a cost of memory and
69   * CPU cycles. If this filter is mapped for static content, then use of efficient direct NIO may be
70   * prevented, thus use of the gzip mechanism of the {@link org.eclipse.jetty.servlet.DefaultServlet} is
71   * advised instead.
72   * </p>
73   * <p>
74   * This filter extends {@link UserAgentFilter} and if the the initParameter <code>excludedAgents</code>
75   * is set to a comma separated list of user agents, then these agents will be excluded from gzip content.
76   * </p>
77   * <p>Init Parameters:</p>
78   * <dl>
79   * <dt>bufferSize</dt>       <dd>The output buffer size. Defaults to 8192. Be careful as values <= 0 will lead to an
80   *                            {@link IllegalArgumentException}.
81   *                            See: {@link java.util.zip.GZIPOutputStream#GZIPOutputStream(java.io.OutputStream, int)}
82   *                            and: {@link java.util.zip.DeflaterOutputStream#DeflaterOutputStream(java.io.OutputStream, Deflater, int)}
83   * </dd>
84   * <dt>minGzipSize</dt>       <dd>Content will only be compressed if content length is either unknown or greater
85   *                            than <code>minGzipSize</code>.
86   * </dd>
87   * <dt>deflateCompressionLevel</dt>       <dd>The compression level used for deflate compression. (0-9).
88   *                            See: {@link java.util.zip.Deflater#Deflater(int, boolean)}
89   * </dd>
90   * <dt>deflateNoWrap</dt>       <dd>The noWrap setting for deflate compression. Defaults to true. (true/false)
91   *                            See: {@link java.util.zip.Deflater#Deflater(int, boolean)}
92   * </dd>
93   * <dt>methods</dt>       <dd>Comma separated list of HTTP methods to compress. If not set, only GET requests are compressed.
94   *  </dd>
95   * <dt>mimeTypes</dt>       <dd>Comma separated list of mime types to compress. If it is not set, then the excludedMimeTypes list is used.
96   * </dd>
97   * <dt>excludedMimeTypes</dt>       <dd>Comma separated list of mime types to never compress. If not set, then the default is the commonly known
98   * image, video, audio and compressed types.
99   * </dd>
100 
101  * <dt>excludedAgents</dt>       <dd>Comma separated list of user agents to exclude from compression. Does a
102  *                            {@link String#contains(CharSequence)} to check if the excluded agent occurs
103  *                            in the user-agent header. If it does -> no compression
104  * </dd>
105  * <dt>excludeAgentPatterns</dt>       <dd>Same as excludedAgents, but accepts regex patterns for more complex matching.
106  * </dd>
107  * <dt>excludePaths</dt>       <dd>Comma separated list of paths to exclude from compression.
108  *                            Does a {@link String#startsWith(String)} comparison to check if the path matches.
109  *                            If it does match -> no compression. To match subpaths use <code>excludePathPatterns</code>
110  *                            instead.
111  * </dd>
112  * <dt>excludePathPatterns</dt>       <dd>Same as excludePath, but accepts regex patterns for more complex matching.
113  * </dd>
114  * <dt>vary</dt>       <dd>Set to the value of the Vary header sent with responses that could be compressed.  By default it is 
115  *                            set to 'Vary: Accept-Encoding, User-Agent' since IE6 is excluded by default from the excludedAgents. 
116  *                            If user-agents are not to be excluded, then this can be set to 'Vary: Accept-Encoding'.  Note also 
117  *                            that shared caches may cache copies of a resource that is varied by User-Agent - one per variation of 
118  *                            the User-Agent, unless the cache does some normalization of the UA string.
119  * </dd>                         
120  * <dt>checkGzExists</dt>       <dd>If set to true, the filter check if a static resource with ".gz" appended exists.  If so then
121  *                            the normal processing is done so that the default servlet can send  the pre existing gz content.
122  *  </dd>
123  *  </dl>
124  */
125 public class GzipFilter extends UserAgentFilter
126 {
127     private static final Logger LOG = Log.getLogger(GzipFilter.class);
128     public final static String GZIP="gzip";
129     public final static String ETAG_GZIP="--gzip\"";
130     public final static String DEFLATE="deflate";
131     public final static String ETAG_DEFLATE="--deflate\"";
132     public final static String ETAG="o.e.j.s.GzipFilter.ETag";
133 
134     protected ServletContext _context;
135     protected final Set<String> _mimeTypes=new HashSet<>();
136     protected boolean _excludeMimeTypes;
137     protected int _bufferSize=8192;
138     protected int _minGzipSize=256;
139     protected int _deflateCompressionLevel=Deflater.DEFAULT_COMPRESSION;
140     protected boolean _deflateNoWrap = true;
141     protected boolean _checkGzExists = true;
142     
143     // non-static, as other GzipFilter instances may have different configurations
144     protected final ThreadLocal<Deflater> _deflater = new ThreadLocal<Deflater>();
145 
146     protected final static ThreadLocal<byte[]> _buffer= new ThreadLocal<byte[]>();
147 
148     protected final Set<String> _methods=new HashSet<String>();
149     protected Set<String> _excludedAgents;
150     protected Set<Pattern> _excludedAgentPatterns;
151     protected Set<String> _excludedPaths;
152     protected Set<Pattern> _excludedPathPatterns;
153     protected String _vary="Accept-Encoding, User-Agent";
154     
155     private static final int STATE_SEPARATOR = 0;
156     private static final int STATE_Q = 1;
157     private static final int STATE_QVALUE = 2;
158     private static final int STATE_DEFAULT = 3;
159 
160 
161     /* ------------------------------------------------------------ */
162     /**
163      * @see org.eclipse.jetty.servlets.UserAgentFilter#init(javax.servlet.FilterConfig)
164      */
165     @Override
166     public void init(FilterConfig filterConfig) throws ServletException
167     {
168         super.init(filterConfig);
169 
170         _context=filterConfig.getServletContext();
171         
172         String tmp=filterConfig.getInitParameter("bufferSize");
173         if (tmp!=null)
174             _bufferSize=Integer.parseInt(tmp);
175 
176         tmp=filterConfig.getInitParameter("minGzipSize");
177         if (tmp!=null)
178             _minGzipSize=Integer.parseInt(tmp);
179 
180         tmp=filterConfig.getInitParameter("deflateCompressionLevel");
181         if (tmp!=null)
182             _deflateCompressionLevel=Integer.parseInt(tmp);
183 
184         tmp=filterConfig.getInitParameter("deflateNoWrap");
185         if (tmp!=null)
186             _deflateNoWrap=Boolean.parseBoolean(tmp);
187 
188         tmp=filterConfig.getInitParameter("checkGzExists");
189         if (tmp!=null)
190             _checkGzExists=Boolean.parseBoolean(tmp);
191         
192         tmp=filterConfig.getInitParameter("methods");
193         if (tmp!=null)
194         {
195             StringTokenizer tok = new StringTokenizer(tmp,",",false);
196             while (tok.hasMoreTokens())
197                 _methods.add(tok.nextToken().trim().toUpperCase(Locale.ENGLISH));
198         }
199         else
200             _methods.add(HttpMethod.GET.asString());
201         
202         tmp=filterConfig.getInitParameter("mimeTypes");
203         if (tmp==null)
204         {
205             _excludeMimeTypes=true;
206             tmp=filterConfig.getInitParameter("excludedMimeTypes");
207             if (tmp==null)
208             {
209                 for (String type:MimeTypes.getKnownMimeTypes())
210                 {
211                     if (type.equals("image/svg+xml")) //always compressable (unless .svgz file)
212                         continue;
213                     if (type.startsWith("image/")||
214                         type.startsWith("audio/")||
215                         type.startsWith("video/"))
216                         _mimeTypes.add(type);
217                 }
218                 _mimeTypes.add("application/compress");
219                 _mimeTypes.add("application/zip");
220                 _mimeTypes.add("application/gzip");
221             }
222             else
223             {
224                 StringTokenizer tok = new StringTokenizer(tmp,",",false);
225                 while (tok.hasMoreTokens())
226                     _mimeTypes.add(tok.nextToken().trim());
227             }
228         }
229         else
230         {
231             StringTokenizer tok = new StringTokenizer(tmp,",",false);
232             while (tok.hasMoreTokens())
233                 _mimeTypes.add(tok.nextToken().trim());
234         }
235         tmp=filterConfig.getInitParameter("excludedAgents");
236         if (tmp!=null)
237         {
238             _excludedAgents=new HashSet<String>();
239             StringTokenizer tok = new StringTokenizer(tmp,",",false);
240             while (tok.hasMoreTokens())
241                _excludedAgents.add(tok.nextToken().trim());
242         }
243 
244         tmp=filterConfig.getInitParameter("excludeAgentPatterns");
245         if (tmp!=null)
246         {
247             _excludedAgentPatterns=new HashSet<Pattern>();
248             StringTokenizer tok = new StringTokenizer(tmp,",",false);
249             while (tok.hasMoreTokens())
250                 _excludedAgentPatterns.add(Pattern.compile(tok.nextToken().trim()));
251         }
252 
253         tmp=filterConfig.getInitParameter("excludePaths");
254         if (tmp!=null)
255         {
256             _excludedPaths=new HashSet<String>();
257             StringTokenizer tok = new StringTokenizer(tmp,",",false);
258             while (tok.hasMoreTokens())
259                 _excludedPaths.add(tok.nextToken().trim());
260         }
261 
262         tmp=filterConfig.getInitParameter("excludePathPatterns");
263         if (tmp!=null)
264         {
265             _excludedPathPatterns=new HashSet<Pattern>();
266             StringTokenizer tok = new StringTokenizer(tmp,",",false);
267             while (tok.hasMoreTokens())
268                 _excludedPathPatterns.add(Pattern.compile(tok.nextToken().trim()));
269         }
270         
271         tmp=filterConfig.getInitParameter("vary");
272         if (tmp!=null)
273             _vary=tmp;
274     }
275 
276     /* ------------------------------------------------------------ */
277     /**
278      * @see org.eclipse.jetty.servlets.UserAgentFilter#destroy()
279      */
280     @Override
281     public void destroy()
282     {
283     }
284 
285     /* ------------------------------------------------------------ */
286     /**
287      * @see org.eclipse.jetty.servlets.UserAgentFilter#doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain)
288      */
289     @Override
290     public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain)
291         throws IOException, ServletException
292     {
293         HttpServletRequest request=(HttpServletRequest)req;
294         HttpServletResponse response=(HttpServletResponse)res;
295 
296         // If not a supported method or it is an Excluded URI - no Vary because no matter what client, this URI is always excluded
297         String requestURI = request.getRequestURI();
298         if (!_methods.contains(request.getMethod()) || isExcludedPath(requestURI))
299         {
300             super.doFilter(request,response,chain);
301             return;
302         }
303 
304         // Exclude non compressible mime-types known from URI extension. - no Vary because no matter what client, this URI is always excluded
305         if (_mimeTypes.size()>0 && _excludeMimeTypes)
306         {
307             String mimeType = _context.getMimeType(request.getRequestURI());
308 
309             if (mimeType!=null)
310             {
311                 mimeType = MimeTypes.getContentTypeWithoutCharset(mimeType);
312                 if (_mimeTypes.contains(mimeType))
313                 {
314                     // handle normally without setting vary header
315                     super.doFilter(request,response,chain);
316                     return;
317                 }
318             }
319         }
320         
321         //If the Content-Encoding is already set, then we won't compress
322         if (response.getHeader("Content-Encoding") != null)
323         {
324             super.doFilter(request,response,chain);
325             return;
326         }
327 
328         if (_checkGzExists && request.getServletContext()!=null)
329         {
330             String path=request.getServletContext().getRealPath(URIUtil.addPaths(request.getServletPath(),request.getPathInfo()));
331             if (path!=null)
332             {
333                 File gz=new File(path+".gz");
334                 if (gz.exists())
335                 {
336                     // allow default servlet to handle
337                     super.doFilter(request,response,chain);
338                     return;
339                 }
340             }
341         }
342         
343         // Excluded User-Agents
344         String ua = getUserAgent(request);
345         boolean ua_excluded=ua!=null&&isExcludedAgent(ua);
346         
347         // Acceptable compression type
348         String compressionType = ua_excluded?null:selectCompression(request.getHeader("accept-encoding"));
349 
350         // Special handling for etags
351         String etag = request.getHeader("If-None-Match"); 
352         if (etag!=null)
353         {
354             int dd=etag.indexOf("--");
355             if (dd>0)
356                 request.setAttribute(ETAG,etag.substring(0,dd)+(etag.endsWith("\"")?"\"":""));
357         }
358 
359         CompressedResponseWrapper wrappedResponse = createWrappedResponse(request,response,compressionType);
360 
361         boolean exceptional=true;
362         try
363         {
364             super.doFilter(request,wrappedResponse,chain);
365             exceptional=false;
366         }
367         finally
368         {
369             if (request.isAsyncStarted())
370             {
371                 request.getAsyncContext().addListener(new FinishOnCompleteListener(wrappedResponse));
372             }
373             else if (exceptional && !response.isCommitted())
374             {
375                 wrappedResponse.resetBuffer();
376                 wrappedResponse.noCompression();
377             }
378             else
379                 wrappedResponse.finish();
380         }
381     }
382 
383     /* ------------------------------------------------------------ */
384     private String selectCompression(String encodingHeader)
385     {
386         // TODO, this could be a little more robust.
387         // prefer gzip over deflate
388         String compression = null;
389         if (encodingHeader!=null)
390         {
391             
392             String[] encodings = getEncodings(encodingHeader);
393             if (encodings != null)
394             {
395                 for (int i=0; i< encodings.length; i++)
396                 {
397                     if (encodings[i].toLowerCase(Locale.ENGLISH).contains(GZIP))
398                     {
399                         if (isEncodingAcceptable(encodings[i]))
400                         {
401                             compression = GZIP;
402                             break; //prefer Gzip over deflate
403                         }
404                     }
405 
406                     if (encodings[i].toLowerCase(Locale.ENGLISH).contains(DEFLATE))
407                     {
408                         if (isEncodingAcceptable(encodings[i]))
409                         {
410                             compression = DEFLATE; //Keep checking in case gzip is acceptable
411                         }
412                     }
413                 }
414             }
415         }
416         return compression;
417     }
418     
419     
420     private String[] getEncodings (String encodingHeader)
421     {
422         if (encodingHeader == null)
423             return null;
424         return encodingHeader.split(",");
425     }
426     
427     private boolean isEncodingAcceptable(String encoding)
428     {    
429         int state = STATE_DEFAULT;
430         int qvalueIdx = -1;
431         for (int i=0;i<encoding.length();i++)
432         {
433             char c = encoding.charAt(i);
434             switch (state)
435             {
436                 case STATE_DEFAULT:
437                 {
438                     if (';' == c)
439                         state = STATE_SEPARATOR;
440                     break;
441                 }
442                 case STATE_SEPARATOR:
443                 {
444                     if ('q' == c || 'Q' == c)
445                         state = STATE_Q;
446                     break;
447                 }
448                 case STATE_Q:
449                 {
450                     if ('=' == c)
451                         state = STATE_QVALUE;
452                     break;
453                 }
454                 case STATE_QVALUE:
455                 {
456                     if (qvalueIdx < 0 && '0' == c || '1' == c)
457                         qvalueIdx = i;
458                     break;
459                 }
460             }
461         }
462         
463         if (qvalueIdx < 0)
464             return true;
465                
466         if ("0".equals(encoding.substring(qvalueIdx).trim()))
467             return false;
468         return true;
469     }
470     
471 
472     protected CompressedResponseWrapper createWrappedResponse(HttpServletRequest request, HttpServletResponse response, final String compressionType)
473     {
474         CompressedResponseWrapper wrappedResponse = null;
475         wrappedResponse = new CompressedResponseWrapper(request,response)
476         {
477             @Override
478             protected AbstractCompressedStream newCompressedStream(HttpServletRequest request, HttpServletResponse response) throws IOException
479             {
480                 return new AbstractCompressedStream(compressionType,request,this,_vary)
481                 {
482                     private Deflater _allocatedDeflater;
483                     private byte[] _allocatedBuffer;
484 
485                     @Override
486                     protected OutputStream createStream() throws IOException
487                     {
488                         if (compressionType == null)
489                         {
490                             return null;
491                         }
492                         
493                         // acquire deflater instance
494                         _allocatedDeflater = _deflater.get();   
495                         if (_allocatedDeflater==null)
496                             _allocatedDeflater = new Deflater(_deflateCompressionLevel,_deflateNoWrap);
497                         else
498                         {
499                             _deflater.set(null);
500                             _allocatedDeflater.reset();
501                         }
502                         
503                         // acquire buffer
504                         _allocatedBuffer = _buffer.get();
505                         if (_allocatedBuffer==null)
506                             _allocatedBuffer = new byte[_bufferSize];
507                         else
508                             _buffer.set(null);
509                         
510                         switch (compressionType)
511                         {
512                             case GZIP:
513                                 return new GzipOutputStream(_response.getOutputStream(),_allocatedDeflater,_allocatedBuffer);
514                             case DEFLATE:
515                                 return new DeflatedOutputStream(_response.getOutputStream(),_allocatedDeflater,_allocatedBuffer);
516                         }
517                         throw new IllegalStateException(compressionType + " not supported");
518                     }
519 
520                     @Override
521                     public void finish() throws IOException
522                     {
523                         super.finish();
524                         if (_allocatedDeflater != null && _deflater.get() == null)
525                         {
526                             _deflater.set(_allocatedDeflater);
527                         }
528                         if (_allocatedBuffer != null && _buffer.get() == null)
529                         {
530                             _buffer.set(_allocatedBuffer);
531                         }
532                     }
533                 };
534             }
535         };
536         configureWrappedResponse(wrappedResponse);
537         return wrappedResponse;
538     }
539 
540     protected void configureWrappedResponse(CompressedResponseWrapper wrappedResponse)
541     {
542         wrappedResponse.setMimeTypes(_mimeTypes,_excludeMimeTypes);
543         wrappedResponse.setBufferSize(_bufferSize);
544         wrappedResponse.setMinCompressSize(_minGzipSize);
545     }
546 
547     private class FinishOnCompleteListener implements AsyncListener
548     {    
549         private CompressedResponseWrapper wrappedResponse;
550 
551         public FinishOnCompleteListener(CompressedResponseWrapper wrappedResponse)
552         {
553             this.wrappedResponse = wrappedResponse;
554         }
555 
556         @Override
557         public void onComplete(AsyncEvent event) throws IOException
558         {          
559             try
560             {
561                 wrappedResponse.finish();
562             }
563             catch (IOException e)
564             {
565                 LOG.warn(e);
566             }
567         }
568 
569         @Override
570         public void onTimeout(AsyncEvent event) throws IOException
571         {
572         }
573 
574         @Override
575         public void onError(AsyncEvent event) throws IOException
576         {
577         }
578 
579         @Override
580         public void onStartAsync(AsyncEvent event) throws IOException
581         {
582         }
583     }
584 
585     /**
586      * Checks to see if the userAgent is excluded
587      *
588      * @param ua
589      *            the user agent
590      * @return boolean true if excluded
591      */
592     private boolean isExcludedAgent(String ua)
593     {
594         if (ua == null)
595             return false;
596 
597         if (_excludedAgents != null)
598         {
599             if (_excludedAgents.contains(ua))
600             {
601                 return true;
602             }
603         }
604         if (_excludedAgentPatterns != null)
605         {
606             for (Pattern pattern : _excludedAgentPatterns)
607             {
608                 if (pattern.matcher(ua).matches())
609                 {
610                     return true;
611                 }
612             }
613         }
614 
615         return false;
616     }
617 
618     /**
619      * Checks to see if the path is excluded
620      *
621      * @param requestURI
622      *            the request uri
623      * @return boolean true if excluded
624      */
625     private boolean isExcludedPath(String requestURI)
626     {
627         if (requestURI == null)
628             return false;
629         if (_excludedPaths != null)
630         {
631             for (String excludedPath : _excludedPaths)
632             {
633                 if (requestURI.startsWith(excludedPath))
634                 {
635                     return true;
636                 }
637             }
638         }
639         if (_excludedPathPatterns != null)
640         {
641             for (Pattern pattern : _excludedPathPatterns)
642             {
643                 if (pattern.matcher(requestURI).matches())
644                 {
645                     return true;
646                 }
647             }
648         }
649         return false;
650     }
651 }