cowboy_compress_h.erl (9396B)
1 %% Copyright (c) 2017, Loïc Hoguin <essen@ninenines.eu> 2 %% 3 %% Permission to use, copy, modify, and/or distribute this software for any 4 %% purpose with or without fee is hereby granted, provided that the above 5 %% copyright notice and this permission notice appear in all copies. 6 %% 7 %% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 %% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 %% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 %% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 %% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 %% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 %% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 15 -module(cowboy_compress_h). 16 -behavior(cowboy_stream). 17 18 -export([init/3]). 19 -export([data/4]). 20 -export([info/3]). 21 -export([terminate/3]). 22 -export([early_error/5]). 23 24 -record(state, { 25 next :: any(), 26 threshold :: non_neg_integer() | undefined, 27 compress = undefined :: undefined | gzip, 28 deflate = undefined :: undefined | zlib:zstream(), 29 deflate_flush = sync :: none | sync 30 }). 31 32 -spec init(cowboy_stream:streamid(), cowboy_req:req(), cowboy:opts()) 33 -> {cowboy_stream:commands(), #state{}}. 34 init(StreamID, Req, Opts) -> 35 State0 = check_req(Req), 36 CompressThreshold = maps:get(compress_threshold, Opts, 300), 37 DeflateFlush = buffering_to_zflush(maps:get(compress_buffering, Opts, false)), 38 {Commands0, Next} = cowboy_stream:init(StreamID, Req, Opts), 39 fold(Commands0, State0#state{next=Next, 40 threshold=CompressThreshold, 41 deflate_flush=DeflateFlush}). 42 43 -spec data(cowboy_stream:streamid(), cowboy_stream:fin(), cowboy_req:resp_body(), State) 44 -> {cowboy_stream:commands(), State} when State::#state{}. 45 data(StreamID, IsFin, Data, State0=#state{next=Next0}) -> 46 {Commands0, Next} = cowboy_stream:data(StreamID, IsFin, Data, Next0), 47 fold(Commands0, State0#state{next=Next}). 48 49 -spec info(cowboy_stream:streamid(), any(), State) 50 -> {cowboy_stream:commands(), State} when State::#state{}. 51 info(StreamID, Info, State0=#state{next=Next0}) -> 52 {Commands0, Next} = cowboy_stream:info(StreamID, Info, Next0), 53 fold(Commands0, State0#state{next=Next}). 54 55 -spec terminate(cowboy_stream:streamid(), cowboy_stream:reason(), #state{}) -> any(). 56 terminate(StreamID, Reason, #state{next=Next, deflate=Z}) -> 57 %% Clean the zlib:stream() in case something went wrong. 58 %% In the normal scenario the stream is already closed. 59 case Z of 60 undefined -> ok; 61 _ -> zlib:close(Z) 62 end, 63 cowboy_stream:terminate(StreamID, Reason, Next). 64 65 -spec early_error(cowboy_stream:streamid(), cowboy_stream:reason(), 66 cowboy_stream:partial_req(), Resp, cowboy:opts()) -> Resp 67 when Resp::cowboy_stream:resp_command(). 68 early_error(StreamID, Reason, PartialReq, Resp, Opts) -> 69 cowboy_stream:early_error(StreamID, Reason, PartialReq, Resp, Opts). 70 71 %% Internal. 72 73 %% Check if the client supports decoding of gzip responses. 74 %% 75 %% A malformed accept-encoding header is ignored (no compression). 76 check_req(Req) -> 77 try cowboy_req:parse_header(<<"accept-encoding">>, Req) of 78 %% Client doesn't support any compression algorithm. 79 undefined -> 80 #state{compress=undefined}; 81 Encodings -> 82 %% We only support gzip so look for it specifically. 83 %% @todo A recipient SHOULD consider "x-gzip" to be 84 %% equivalent to "gzip". (RFC7230 4.2.3) 85 case [E || E={<<"gzip">>, Q} <- Encodings, Q =/= 0] of 86 [] -> 87 #state{compress=undefined}; 88 _ -> 89 #state{compress=gzip} 90 end 91 catch 92 _:_ -> 93 #state{compress=undefined} 94 end. 95 96 %% Do not compress responses that contain the content-encoding header. 97 check_resp_headers(#{<<"content-encoding">> := _}, State) -> 98 State#state{compress=undefined}; 99 check_resp_headers(_, State) -> 100 State. 101 102 fold(Commands, State=#state{compress=undefined}) -> 103 {Commands, State}; 104 fold(Commands, State) -> 105 fold(Commands, State, []). 106 107 fold([], State, Acc) -> 108 {lists:reverse(Acc), State}; 109 %% We do not compress full sendfile bodies. 110 fold([Response={response, _, _, {sendfile, _, _, _}}|Tail], State, Acc) -> 111 fold(Tail, State, [Response|Acc]); 112 %% We compress full responses directly, unless they are lower than 113 %% the configured threshold or we find we are not able to by looking at the headers. 114 fold([Response0={response, _, Headers, Body}|Tail], 115 State0=#state{threshold=CompressThreshold}, Acc) -> 116 case check_resp_headers(Headers, State0) of 117 State=#state{compress=undefined} -> 118 fold(Tail, State, [Response0|Acc]); 119 State1 -> 120 BodyLength = iolist_size(Body), 121 if 122 BodyLength =< CompressThreshold -> 123 fold(Tail, State1, [Response0|Acc]); 124 true -> 125 {Response, State} = gzip_response(Response0, State1), 126 fold(Tail, State, [Response|Acc]) 127 end 128 end; 129 %% Check headers and initiate compression... 130 fold([Response0={headers, _, Headers}|Tail], State0, Acc) -> 131 case check_resp_headers(Headers, State0) of 132 State=#state{compress=undefined} -> 133 fold(Tail, State, [Response0|Acc]); 134 State1 -> 135 {Response, State} = gzip_headers(Response0, State1), 136 fold(Tail, State, [Response|Acc]) 137 end; 138 %% then compress each data commands individually. 139 fold([Data0={data, _, _}|Tail], State0=#state{compress=gzip}, Acc) -> 140 {Data, State} = gzip_data(Data0, State0), 141 fold(Tail, State, [Data|Acc]); 142 %% When trailers are sent we need to end the compression. 143 %% This results in an extra data command being sent. 144 fold([Trailers={trailers, _}|Tail], State0=#state{compress=gzip}, Acc) -> 145 {{data, fin, Data}, State} = gzip_data({data, fin, <<>>}, State0), 146 fold(Tail, State, [Trailers, {data, nofin, Data}|Acc]); 147 %% All the options from this handler can be updated for the current stream. 148 %% The set_options command must be propagated as-is regardless. 149 fold([SetOptions={set_options, Opts}|Tail], State=#state{ 150 threshold=CompressThreshold0, deflate_flush=DeflateFlush0}, Acc) -> 151 CompressThreshold = maps:get(compress_threshold, Opts, CompressThreshold0), 152 DeflateFlush = case Opts of 153 #{compress_buffering := CompressBuffering} -> 154 buffering_to_zflush(CompressBuffering); 155 _ -> 156 DeflateFlush0 157 end, 158 fold(Tail, State#state{threshold=CompressThreshold, deflate_flush=DeflateFlush}, 159 [SetOptions|Acc]); 160 %% Otherwise, we have an unrelated command or compression is disabled. 161 fold([Command|Tail], State, Acc) -> 162 fold(Tail, State, [Command|Acc]). 163 164 buffering_to_zflush(true) -> none; 165 buffering_to_zflush(false) -> sync. 166 167 gzip_response({response, Status, Headers, Body}, State) -> 168 %% We can't call zlib:gzip/1 because it does an 169 %% iolist_to_binary(GzBody) at the end to return 170 %% a binary(). Therefore the code here is largely 171 %% a duplicate of the code of that function. 172 Z = zlib:open(), 173 GzBody = try 174 %% 31 = 16+?MAX_WBITS from zlib.erl 175 %% @todo It might be good to allow them to be configured? 176 zlib:deflateInit(Z, default, deflated, 31, 8, default), 177 Gz = zlib:deflate(Z, Body, finish), 178 zlib:deflateEnd(Z), 179 Gz 180 after 181 zlib:close(Z) 182 end, 183 {{response, Status, vary(Headers#{ 184 <<"content-length">> => integer_to_binary(iolist_size(GzBody)), 185 <<"content-encoding">> => <<"gzip">> 186 }), GzBody}, State}. 187 188 gzip_headers({headers, Status, Headers0}, State) -> 189 Z = zlib:open(), 190 %% We use the same arguments as when compressing the body fully. 191 %% @todo It might be good to allow them to be configured? 192 zlib:deflateInit(Z, default, deflated, 31, 8, default), 193 Headers = maps:remove(<<"content-length">>, Headers0), 194 {{headers, Status, vary(Headers#{ 195 <<"content-encoding">> => <<"gzip">> 196 })}, State#state{deflate=Z}}. 197 198 %% We must add content-encoding to vary if it's not already there. 199 vary(Headers=#{<<"vary">> := Vary}) -> 200 try cow_http_hd:parse_vary(iolist_to_binary(Vary)) of 201 '*' -> Headers; 202 List -> 203 case lists:member(<<"accept-encoding">>, List) of 204 true -> Headers; 205 false -> Headers#{<<"vary">> => [Vary, <<", accept-encoding">>]} 206 end 207 catch _:_ -> 208 %% The vary header is invalid. Probably empty. We replace it with ours. 209 Headers#{<<"vary">> => <<"accept-encoding">>} 210 end; 211 vary(Headers) -> 212 Headers#{<<"vary">> => <<"accept-encoding">>}. 213 214 %% It is not possible to combine zlib and the sendfile 215 %% syscall as far as I can tell, because the zlib format 216 %% includes a checksum at the end of the stream. We have 217 %% to read the file in memory, making this not suitable for 218 %% large files. 219 gzip_data({data, nofin, Sendfile={sendfile, _, _, _}}, 220 State=#state{deflate=Z, deflate_flush=Flush}) -> 221 {ok, Data0} = read_file(Sendfile), 222 Data = zlib:deflate(Z, Data0, Flush), 223 {{data, nofin, Data}, State}; 224 gzip_data({data, fin, Sendfile={sendfile, _, _, _}}, State=#state{deflate=Z}) -> 225 {ok, Data0} = read_file(Sendfile), 226 Data = zlib:deflate(Z, Data0, finish), 227 zlib:deflateEnd(Z), 228 zlib:close(Z), 229 {{data, fin, Data}, State#state{deflate=undefined}}; 230 gzip_data({data, nofin, Data0}, State=#state{deflate=Z, deflate_flush=Flush}) -> 231 Data = zlib:deflate(Z, Data0, Flush), 232 {{data, nofin, Data}, State}; 233 gzip_data({data, fin, Data0}, State=#state{deflate=Z}) -> 234 Data = zlib:deflate(Z, Data0, finish), 235 zlib:deflateEnd(Z), 236 zlib:close(Z), 237 {{data, fin, Data}, State#state{deflate=undefined}}. 238 239 read_file({sendfile, Offset, Bytes, Path}) -> 240 {ok, IoDevice} = file:open(Path, [read, raw, binary]), 241 try 242 _ = case Offset of 243 0 -> ok; 244 _ -> file:position(IoDevice, {bof, Offset}) 245 end, 246 file:read(IoDevice, Bytes) 247 after 248 file:close(IoDevice) 249 end.