Mercurial > hg > nginx

comparison src/http/modules/ngx_http_gzip_filter.c @ 156:afc333135a6b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
nginx-0.0.1-2003-10-23-10:13:16 import
author Igor Sysoev <igor@sysoev.ru>
date Thu, 23 Oct 2003 06:13:16 +0000
parents 46eb23d9471d
children 70b36c805682
comparison
equal deleted inserted replaced
155:46eb23d9471d 156:afc333135a6b
5 5
6 #include <zlib.h> 6 #include <zlib.h>
7 7
8 8
9 typedef struct { 9 typedef struct {
10 int enable; 10 int enable;
11 ngx_bufs_t bufs; 11 ngx_bufs_t bufs;
12 int no_buffer; 12 int level;
13 int wbits;
14 int memlevel;
15 int no_buffer;
13 } ngx_http_gzip_conf_t; 16 } ngx_http_gzip_conf_t;
14 17
15 18
16 typedef struct { 19 typedef struct {
17 ngx_chain_t *in; 20 ngx_chain_t *in;
18 ngx_chain_t *free; 21 ngx_chain_t *free;
19 ngx_chain_t *busy; 22 ngx_chain_t *busy;
20 ngx_chain_t *out; 23 ngx_chain_t *out;
21 ngx_chain_t **last_out; 24 ngx_chain_t **last_out;
22 ngx_hunk_t *in_hunk; 25 ngx_hunk_t *in_hunk;
23 ngx_hunk_t *out_hunk; 26 ngx_hunk_t *out_hunk;
24 int hunks; 27 int hunks;
25 28
26 off_t length; 29 off_t length;
27 void *alloc; 30
28 31 void *preallocated;
29 unsigned flush:4; 32 char *free_mem;
30 unsigned redo:1; 33 int allocated;
31 34
32 u_int crc32; 35 unsigned flush:4;
33 z_stream zstream; 36 unsigned redo:1;
37
38 u_int crc32;
39 z_stream zstream;
40 ngx_http_request_t *request;
34 } ngx_http_gzip_ctx_t; 41 } ngx_http_gzip_ctx_t;
35 42
43
44 static void *ngx_http_gzip_filter_alloc(void *opaque, u_int items,
45 u_int size);
46 static void ngx_http_gzip_filter_free(void *opaque, void *address);
36 47
37 ngx_inline static int ngx_http_gzip_error(ngx_http_gzip_ctx_t *ctx); 48 ngx_inline static int ngx_http_gzip_error(ngx_http_gzip_ctx_t *ctx);
38 static int ngx_http_gzip_filter_init(ngx_cycle_t *cycle); 49 static int ngx_http_gzip_filter_init(ngx_cycle_t *cycle);
39 static void *ngx_http_gzip_create_conf(ngx_conf_t *cf); 50 static void *ngx_http_gzip_create_conf(ngx_conf_t *cf);
40 static char *ngx_http_gzip_merge_conf(ngx_conf_t *cf, 51 static char *ngx_http_gzip_merge_conf(ngx_conf_t *cf,
41 void *parent, void *child); 52 void *parent, void *child);
53 static char *ngx_http_gzip_set_window(ngx_conf_t *cf, ngx_command_t *cmd,
54 void *conf);
55 static char *ngx_http_gzip_set_hash(ngx_conf_t *cf, ngx_command_t *cmd,
56 void *conf);
57
58 static ngx_conf_bounds_t ngx_http_gzip_comp_level_bounds;
42 59
43 60
44 static ngx_command_t ngx_http_gzip_filter_commands[] = { 61 static ngx_command_t ngx_http_gzip_filter_commands[] = {
45 62
46 {ngx_string("gzip"), 63 {ngx_string("gzip"),
53 {ngx_string("gzip_buffers"), 70 {ngx_string("gzip_buffers"),
54 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE2, 71 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE2,
55 ngx_conf_set_bufs_slot, 72 ngx_conf_set_bufs_slot,
56 NGX_HTTP_LOC_CONF_OFFSET, 73 NGX_HTTP_LOC_CONF_OFFSET,
57 offsetof(ngx_http_gzip_conf_t, bufs), 74 offsetof(ngx_http_gzip_conf_t, bufs),
75 NULL},
76
77 {ngx_string("gzip_comp_level"),
78 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
79 ngx_conf_set_num_slot,
80 NGX_HTTP_LOC_CONF_OFFSET,
81 offsetof(ngx_http_gzip_conf_t, level),
82 &ngx_http_gzip_comp_level_bounds},
83
84 {ngx_string("gzip_window"),
85 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
86 ngx_http_gzip_set_window,
87 NGX_HTTP_LOC_CONF_OFFSET,
88 offsetof(ngx_http_gzip_conf_t, wbits),
89 NULL},
90
91 {ngx_string("gzip_hash"),
92 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
93 ngx_http_gzip_set_hash,
94 NGX_HTTP_LOC_CONF_OFFSET,
95 offsetof(ngx_http_gzip_conf_t, memlevel),
58 NULL}, 96 NULL},
59 97
60 {ngx_string("gzip_no_buffer"), 98 {ngx_string("gzip_no_buffer"),
61 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, 99 NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
62 ngx_conf_set_flag_slot, 100 ngx_conf_set_flag_slot,
88 ngx_http_gzip_filter_init, /* init module */ 126 ngx_http_gzip_filter_init, /* init module */
89 NULL /* init child */ 127 NULL /* init child */
90 }; 128 };
91 129
92 130
93 static char gzheader[10] = { 0x1f, 131 static ngx_conf_bounds_t ngx_http_gzip_comp_level_bounds = {
94 (char) 0x8b, /* suppress MSVC warning */ 132 ngx_conf_check_num_bounds, { { 1, 9 } }
95 Z_DEFLATED, 0, 0, 0, 0, 0, 0, 3 }; 133 };
134
135
136 static u_char gzheader[10] = { 0x1f, 0x8b, Z_DEFLATED, 0, 0, 0, 0, 0, 0, 3 };
96 137
97 #if (HAVE_LITTLE_ENDIAN) 138 #if (HAVE_LITTLE_ENDIAN)
98 139
99 struct gztrailer { 140 struct gztrailer {
100 u_int crc32; 141 u_int crc32;
102 }; 143 };
103 144
104 #else /* HAVE_BIG_ENDIAN */ 145 #else /* HAVE_BIG_ENDIAN */
105 146
106 struct gztrailer { 147 struct gztrailer {
107 unsigned char crc32[4]; 148 u_char crc32[4];
108 unsigned char zlen[4]; 149 u_char zlen[4];
109 }; 150 };
110 151
111 #endif 152 #endif
112 153
113 154
144 return ngx_http_next_header_filter(r); 185 return ngx_http_next_header_filter(r);
145 } 186 }
146 187
147 ngx_http_create_ctx(r, ctx, ngx_http_gzip_filter_module, 188 ngx_http_create_ctx(r, ctx, ngx_http_gzip_filter_module,
148 sizeof(ngx_http_gzip_ctx_t), NGX_ERROR); 189 sizeof(ngx_http_gzip_ctx_t), NGX_ERROR);
190 ctx->request = r;
149 191
150 ngx_test_null(r->headers_out.content_encoding, 192 ngx_test_null(r->headers_out.content_encoding,
151 ngx_push_table(r->headers_out.headers), 193 ngx_push_table(r->headers_out.headers),
152 NGX_ERROR); 194 NGX_ERROR);
153 195
165 } 207 }
166 208
167 209
168 static int ngx_http_gzip_body_filter(ngx_http_request_t *r, ngx_chain_t *in) 210 static int ngx_http_gzip_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
169 { 211 {
170 int rc, wbits, mem_level, zin, zout, last; 212 int rc, wbits, memlevel, zin, zout, last;
171 struct gztrailer *trailer; 213 struct gztrailer *trailer;
172 ngx_hunk_t *h; 214 ngx_hunk_t *h;
173 ngx_chain_t *cl; 215 ngx_chain_t *cl;
174 ngx_http_gzip_ctx_t *ctx; 216 ngx_http_gzip_ctx_t *ctx;
175 ngx_http_gzip_conf_t *conf; 217 ngx_http_gzip_conf_t *conf;
180 return ngx_http_next_body_filter(r, in); 222 return ngx_http_next_body_filter(r, in);
181 } 223 }
182 224
183 conf = ngx_http_get_module_loc_conf(r, ngx_http_gzip_filter_module); 225 conf = ngx_http_get_module_loc_conf(r, ngx_http_gzip_filter_module);
184 226
185 if (ctx->alloc == NULL) { 227 if (ctx->preallocated == NULL) {
186 wbits = MAX_WBITS; 228 wbits = conf->wbits;
187 mem_level = MAX_MEM_LEVEL - 1; 229 memlevel = conf->memlevel;
188 230
189 if (ctx->length > 0) { 231 if (ctx->length > 0) {
190 232
191 /* the actual zlib window size is smaller by 262 bytes */ 233 /* the actual zlib window size is smaller by 262 bytes */
192 234
193 while (ctx->length < ((1 << (wbits - 1)) - 262)) { 235 while (ctx->length < ((1 << (wbits - 1)) - 262)) {
194 wbits--; 236 wbits--;
195 mem_level--; 237 memlevel--;
196 } 238 }
197 } 239 }
198 240
199 #if 0 241 /*
200 ngx_test_null(ctx->alloc, ngx_alloc(200K, r->log), NGX_ERROR); 242 * We preallocate a memory for zlib in one hunk (200K-400K), this
201 #else 243 * dicreases number of malloc() and free() calls and probably
202 ctx->alloc = (void *) -1; 244 * syscalls.
203 #endif 245 * Besides we free() this memory as soon as the gzipping will complete
204 246 * and do not wait while a whole response will be sent to a client.
205 rc = deflateInit2(&ctx->zstream, /**/ 1, Z_DEFLATED, 247 *
206 -wbits, mem_level, Z_DEFAULT_STRATEGY); 248 * 8K is for zlib deflate_state (~6K).
249 *
250 * TODO: 64-bit, round to PAGE_SIZE
251 */
252
253 ctx->allocated = 8192 + (1 << (wbits + 2)) + (1 << (memlevel + 9));
254
255 ngx_test_null(ctx->preallocated, ngx_palloc(r->pool, ctx->allocated),
256 NGX_ERROR);
257 ctx->free_mem = ctx->preallocated;
258
259 ctx->zstream.zalloc = ngx_http_gzip_filter_alloc;
260 ctx->zstream.zfree = ngx_http_gzip_filter_free;
261 ctx->zstream.opaque = ctx;
262
263 rc = deflateInit2(&ctx->zstream, conf->level, Z_DEFLATED,
264 -wbits, memlevel, Z_DEFAULT_STRATEGY);
207 265
208 if (rc != Z_OK) { 266 if (rc != Z_OK) {
209 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, 267 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
210 "deflateInit2() failed: %d", rc); 268 "deflateInit2() failed: %d", rc);
211 return ngx_http_gzip_error(ctx); 269 return ngx_http_gzip_error(ctx);
212 } 270 }
213 271
214 ngx_test_null(h, ngx_calloc_hunk(r->pool), ngx_http_gzip_error(ctx)); 272 ngx_test_null(h, ngx_calloc_hunk(r->pool), ngx_http_gzip_error(ctx));
215 273
216 h->type = NGX_HUNK_IN_MEMORY|NGX_HUNK_MEMORY; 274 h->type = NGX_HUNK_IN_MEMORY|NGX_HUNK_MEMORY;
217 h->pos = gzheader; 275 h->pos = (char *) gzheader;
218 h->last = h->pos + 10; 276 h->last = h->pos + 10;
219 277
220 ngx_alloc_link_and_set_hunk(cl, h, r->pool, ngx_http_gzip_error(ctx)); 278 ngx_alloc_link_and_set_hunk(cl, h, r->pool, ngx_http_gzip_error(ctx));
221 ctx->out = cl; 279 ctx->out = cl;
222 ctx->last_out = &cl->next; 280 ctx->last_out = &cl->next;
248 } 306 }
249 307
250 ctx->in_hunk = ctx->in->hunk; 308 ctx->in_hunk = ctx->in->hunk;
251 ctx->in = ctx->in->next; 309 ctx->in = ctx->in->next;
252 310
253 ctx->zstream.next_in = (unsigned char *) ctx->in_hunk->pos; 311 ctx->zstream.next_in = (u_char *) ctx->in_hunk->pos;
254 ctx->zstream.avail_in = ctx->in_hunk->last - ctx->in_hunk->pos; 312 ctx->zstream.avail_in = ctx->in_hunk->last - ctx->in_hunk->pos;
255 313
256 if (ctx->in_hunk->type & NGX_HUNK_LAST) { 314 if (ctx->in_hunk->type & NGX_HUNK_LAST) {
257 ctx->flush = Z_FINISH; 315 ctx->flush = Z_FINISH;
258 316
290 348
291 } else { 349 } else {
292 break; 350 break;
293 } 351 }
294 352
295 ctx->zstream.next_out = (unsigned char *) ctx->out_hunk->pos; 353 ctx->zstream.next_out = (u_char *) ctx->out_hunk->pos;
296 ctx->zstream.avail_out = conf->bufs.size; 354 ctx->zstream.avail_out = conf->bufs.size;
297 } 355 }
298 356
299 ngx_log_debug(r->connection->log, "deflate(): %08x %08x %d %d %d" _ 357 ngx_log_debug(r->connection->log, "deflate(): %08x %08x %d %d %d" _
300 ctx->zstream.next_in _ ctx->zstream.next_out _ 358 ctx->zstream.next_in _ ctx->zstream.next_out _
335 ctx->last_out = &cl->next; 393 ctx->last_out = &cl->next;
336 394
337 break; 395 break;
338 396
339 } else if (ctx->flush == Z_FINISH) { 397 } else if (ctx->flush == Z_FINISH) {
398
340 /* rc == Z_STREAM_END */ 399 /* rc == Z_STREAM_END */
341 400
342 zin = ctx->zstream.total_in; 401 zin = ctx->zstream.total_in;
343 zout = 10 + ctx->zstream.total_out + 8; 402 zout = 10 + ctx->zstream.total_out + 8;
344 403
346 if (rc != Z_OK) { 405 if (rc != Z_OK) {
347 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, 406 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
348 "deflateEnd() failed: %d", rc); 407 "deflateEnd() failed: %d", rc);
349 return ngx_http_gzip_error(ctx); 408 return ngx_http_gzip_error(ctx);
350 } 409 }
410
411 ngx_pfree(r->pool, ctx->preallocated);
351 412
352 ctx->flush = Z_NO_FLUSH; 413 ctx->flush = Z_NO_FLUSH;
353 414
354 ngx_alloc_link_and_set_hunk(cl, ctx->out_hunk, r->pool, 415 ngx_alloc_link_and_set_hunk(cl, ctx->out_hunk, r->pool,
355 ngx_http_gzip_error(ctx)); 416 ngx_http_gzip_error(ctx));
383 /* STUB */ 444 /* STUB */
384 #endif 445 #endif
385 446
386 ctx->zstream.avail_in = 0; 447 ctx->zstream.avail_in = 0;
387 ctx->zstream.avail_out = 0; 448 ctx->zstream.avail_out = 0;
388 #if 0 449
389 ngx_free(ctx->alloc);
390 #endif
391 ngx_http_delete_ctx(r, ngx_http_gzip_filter_module); 450 ngx_http_delete_ctx(r, ngx_http_gzip_filter_module);
392 451
393 break; 452 break;
394 453
395 } else if (conf->no_buffer && ctx->in == NULL) { 454 } else if (conf->no_buffer && ctx->in == NULL) {
418 ctx->last_out = &ctx->out; 477 ctx->last_out = &ctx->out;
419 } 478 }
420 } 479 }
421 480
422 481
482 static void *ngx_http_gzip_filter_alloc(void *opaque, u_int items, u_int size)
483 {
484 ngx_http_gzip_ctx_t *ctx = opaque;
485
486 int alloc;
487 void *p;
488
489
490 alloc = items * size;
491 if (alloc % 512 != 0) {
492
493 /* we allocate 8K for zlib deflate_state (~6K) */
494 /* TODO: PAGE_SIZE */
495
496 alloc = (alloc + 4095) & ~4095;
497 }
498
499 if (alloc <= ctx->allocated) {
500 p = ctx->free_mem;
501 ctx->free_mem += alloc;
502 ctx->allocated -= alloc;
503
504 #if 0
505 ngx_log_debug(ctx->request->connection->log, "ALLOC: %d:%d:%d:%08X" _
506 items _ size _ alloc _ p);
507 #endif
508
509 return p;
510 }
511
512 ngx_log_error(NGX_LOG_ALERT, ctx->request->connection->log, 0,
513 "gzip filter failed to use preallocated memory: %d of %d",
514 items * size, ctx->allocated);
515
516 p = ngx_palloc(ctx->request->pool, items * size);
517
518 return p;
519 }
520
521
522 static void ngx_http_gzip_filter_free(void *opaque, void *address)
523 {
524 ngx_http_gzip_ctx_t *ctx = opaque;
525
526 #if 0
527 ngx_log_debug(ctx->request->connection->log, "FREE: %08X" _ address);
528 #endif
529 }
530
531
423 ngx_inline static int ngx_http_gzip_error(ngx_http_gzip_ctx_t *ctx) 532 ngx_inline static int ngx_http_gzip_error(ngx_http_gzip_ctx_t *ctx)
424 { 533 {
425 #if 0
426 ngx_free(ctx->alloc);
427 #else
428 deflateEnd(&ctx->zstream); 534 deflateEnd(&ctx->zstream);
429 #endif 535
536 ngx_pfree(ctx->request->pool, ctx->preallocated);
430 537
431 ctx->zstream.avail_in = 0; 538 ctx->zstream.avail_in = 0;
432 ctx->zstream.avail_out = 0; 539 ctx->zstream.avail_out = 0;
433 540
434 return NGX_ERROR; 541 return NGX_ERROR;
454 ngx_test_null(conf, 561 ngx_test_null(conf,
455 ngx_pcalloc(cf->pool, sizeof(ngx_http_gzip_conf_t)), 562 ngx_pcalloc(cf->pool, sizeof(ngx_http_gzip_conf_t)),
456 NGX_CONF_ERROR); 563 NGX_CONF_ERROR);
457 564
458 conf->enable = NGX_CONF_UNSET; 565 conf->enable = NGX_CONF_UNSET;
459 /* conf->bufs.num = 0; */ 566 /* conf->bufs.num = 0; */
460 conf->no_buffer = NGX_CONF_UNSET; 567 conf->no_buffer = NGX_CONF_UNSET;
568 conf->level = NGX_CONF_UNSET;
569 conf->wbits = NGX_CONF_UNSET;
570 conf->memlevel = NGX_CONF_UNSET;
461 571
462 return conf; 572 return conf;
463 } 573 }
464 574
465 575
470 ngx_http_gzip_conf_t *conf = child; 580 ngx_http_gzip_conf_t *conf = child;
471 581
472 ngx_conf_merge_value(conf->enable, prev->enable, 0); 582 ngx_conf_merge_value(conf->enable, prev->enable, 0);
473 ngx_conf_merge_bufs_value(conf->bufs, prev->bufs, 4, 583 ngx_conf_merge_bufs_value(conf->bufs, prev->bufs, 4,
474 /* STUB: PAGE_SIZE */ 4096); 584 /* STUB: PAGE_SIZE */ 4096);
585 ngx_conf_merge_value(conf->level, prev->level, 1);
586 ngx_conf_merge_value(conf->wbits, prev->wbits, MAX_WBITS);
587 ngx_conf_merge_value(conf->memlevel, prev->memlevel, MAX_MEM_LEVEL - 1);
475 ngx_conf_merge_value(conf->no_buffer, prev->no_buffer, 0); 588 ngx_conf_merge_value(conf->no_buffer, prev->no_buffer, 0);
476 589
477 return NGX_CONF_OK; 590 return NGX_CONF_OK;
478 } 591 }
592
593
594 static char *ngx_http_gzip_set_window(ngx_conf_t *cf, ngx_command_t *cmd,
595 void *conf)
596 {
597 ngx_http_gzip_conf_t *lcf = conf;
598
599 int wbits, wsize;
600 char *rv;
601
602
603 rv = ngx_conf_set_size_slot(cf, cmd, conf);
604 if (rv) {
605 return rv;
606 }
607
608 ngx_conf_log_error(NGX_LOG_INFO, cf, 0, "WBITS: %d", lcf->wbits);
609
610 wbits = 15;
611 for (wsize = 32 * 1024; wsize > 256; wsize >>= 1) {
612
613 if (wsize == lcf->wbits) {
614 lcf->wbits = wbits;
615 ngx_conf_log_error(NGX_LOG_INFO, cf, 0, "WBITS: %d", lcf->wbits);
616 return NULL;
617 }
618
619 wbits--;
620 }
621
622 return "must be 512, 1k, 2k, 4k, 8k, 16k, or 32k";
623 }
624
625
626 static char *ngx_http_gzip_set_hash(ngx_conf_t *cf, ngx_command_t *cmd,
627 void *conf)
628 {
629 ngx_http_gzip_conf_t *lcf = conf;
630
631 int memlevel, hsize;
632 char *rv;
633
634
635 rv = ngx_conf_set_size_slot(cf, cmd, conf);
636 if (rv) {
637 return rv;
638 }
639
640 ngx_conf_log_error(NGX_LOG_INFO, cf, 0, "MEMLEVEL: %d", lcf->memlevel);
641
642 memlevel = 9;
643 for (hsize = 128 * 1024; hsize > 256; hsize >>= 1) {
644
645 if (hsize == lcf->memlevel) {
646 lcf->memlevel = memlevel;
647 ngx_conf_log_error(NGX_LOG_INFO, cf, 0, "MEMLEVEL: %d", lcf->memlevel);
648 return NULL;
649 }
650
651 memlevel--;
652 }
653
654 return "must be 512, 1k, 2k, 4k, 8k, 16k, 32k, 64k, or 128k";
655 }