Mercurial > hg > nginx

comparison src/http/modules/ngx_http_gzip_filter_module.c @ 7534:ac5a741d39cf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Gzip: use zlib to write header and trailer. When nginx is used with zlib patched with [1], which provides integration with the future IBM Z hardware deflate acceleration, it ends up computing CRC32 twice: one time in hardware, which always does this, and one time in software by explicitly calling crc32(). crc32() calls were added in changesets 133:b27548f540ad ("nginx-0.0.1- 2003-09-24-23:51:12 import") and 134:d57c6835225c ("nginx-0.0.1- 2003-09-26-09:45:21 import") as part of gzip wrapping feature - back then zlib did not support it. However, since then gzip wrapping was implemented in zlib v1.2.0.4, and it's already being used by nginx for log compression. This patch replaces hand-written gzip wrapping with the one provided by zlib. It simplifies the code, and makes it avoid computing CRC32 twice when using hardware acceleration. [1] https://github.com/madler/zlib/pull/410
author Ilya Leoshkevich <iii@linux.ibm.com>
date Fri, 12 Jul 2019 12:43:08 +0200
parents 29e9571b1989
children 0ef2bc0ec9c9
comparison
equal deleted inserted replaced
7533:5f642712e7ad 7534:ac5a741d39cf
53 53
54 unsigned flush:4; 54 unsigned flush:4;
55 unsigned redo:1; 55 unsigned redo:1;
56 unsigned done:1; 56 unsigned done:1;
57 unsigned nomem:1; 57 unsigned nomem:1;
58 unsigned gzheader:1;
59 unsigned buffering:1; 58 unsigned buffering:1;
60 unsigned intel:1; 59 unsigned intel:1;
61 60
62 size_t zin; 61 size_t zin;
63 size_t zout; 62 size_t zout;
64 63
65 uint32_t crc32;
66 z_stream zstream; 64 z_stream zstream;
67 ngx_http_request_t *request; 65 ngx_http_request_t *request;
68 } ngx_http_gzip_ctx_t; 66 } ngx_http_gzip_ctx_t;
69
70
71 #if (NGX_HAVE_LITTLE_ENDIAN && NGX_HAVE_NONALIGNED)
72
73 struct gztrailer {
74 uint32_t crc32;
75 uint32_t zlen;
76 };
77
78 #else /* NGX_HAVE_BIG_ENDIAN || !NGX_HAVE_NONALIGNED */
79
80 struct gztrailer {
81 u_char crc32[4];
82 u_char zlen[4];
83 };
84
85 #endif
86 67
87 68
88 static void ngx_http_gzip_filter_memory(ngx_http_request_t *r, 69 static void ngx_http_gzip_filter_memory(ngx_http_request_t *r,
89 ngx_http_gzip_ctx_t *ctx); 70 ngx_http_gzip_ctx_t *ctx);
90 static ngx_int_t ngx_http_gzip_filter_buffer(ngx_http_gzip_ctx_t *ctx, 71 static ngx_int_t ngx_http_gzip_filter_buffer(ngx_http_gzip_ctx_t *ctx,
91 ngx_chain_t *in); 72 ngx_chain_t *in);
92 static ngx_int_t ngx_http_gzip_filter_deflate_start(ngx_http_request_t *r, 73 static ngx_int_t ngx_http_gzip_filter_deflate_start(ngx_http_request_t *r,
93 ngx_http_gzip_ctx_t *ctx);
94 static ngx_int_t ngx_http_gzip_filter_gzheader(ngx_http_request_t *r,
95 ngx_http_gzip_ctx_t *ctx); 74 ngx_http_gzip_ctx_t *ctx);
96 static ngx_int_t ngx_http_gzip_filter_add_data(ngx_http_request_t *r, 75 static ngx_int_t ngx_http_gzip_filter_add_data(ngx_http_request_t *r,
97 ngx_http_gzip_ctx_t *ctx); 76 ngx_http_gzip_ctx_t *ctx);
98 static ngx_int_t ngx_http_gzip_filter_get_buf(ngx_http_request_t *r, 77 static ngx_int_t ngx_http_gzip_filter_get_buf(ngx_http_request_t *r,
99 ngx_http_gzip_ctx_t *ctx); 78 ngx_http_gzip_ctx_t *ctx);
444 ngx_http_gzip_filter_free_copy_buf(r, ctx); 423 ngx_http_gzip_filter_free_copy_buf(r, ctx);
445 424
446 return ctx->busy ? NGX_AGAIN : NGX_OK; 425 return ctx->busy ? NGX_AGAIN : NGX_OK;
447 } 426 }
448 427
449 if (!ctx->gzheader) {
450 if (ngx_http_gzip_filter_gzheader(r, ctx) != NGX_OK) {
451 goto failed;
452 }
453 }
454
455 rc = ngx_http_next_body_filter(r, ctx->out); 428 rc = ngx_http_next_body_filter(r, ctx->out);
456 429
457 if (rc == NGX_ERROR) { 430 if (rc == NGX_ERROR) {
458 goto failed; 431 goto failed;
459 } 432 }
641 ctx->zstream.zalloc = ngx_http_gzip_filter_alloc; 614 ctx->zstream.zalloc = ngx_http_gzip_filter_alloc;
642 ctx->zstream.zfree = ngx_http_gzip_filter_free; 615 ctx->zstream.zfree = ngx_http_gzip_filter_free;
643 ctx->zstream.opaque = ctx; 616 ctx->zstream.opaque = ctx;
644 617
645 rc = deflateInit2(&ctx->zstream, (int) conf->level, Z_DEFLATED, 618 rc = deflateInit2(&ctx->zstream, (int) conf->level, Z_DEFLATED,
646 - ctx->wbits, ctx->memlevel, Z_DEFAULT_STRATEGY); 619 ctx->wbits + 16, ctx->memlevel, Z_DEFAULT_STRATEGY);
647 620
648 if (rc != Z_OK) { 621 if (rc != Z_OK) {
649 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, 622 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
650 "deflateInit2() failed: %d", rc); 623 "deflateInit2() failed: %d", rc);
651 return NGX_ERROR; 624 return NGX_ERROR;
652 } 625 }
653 626
654 ctx->last_out = &ctx->out; 627 ctx->last_out = &ctx->out;
655 ctx->crc32 = crc32(0L, Z_NULL, 0);
656 ctx->flush = Z_NO_FLUSH; 628 ctx->flush = Z_NO_FLUSH;
657
658 return NGX_OK;
659 }
660
661
662 static ngx_int_t
663 ngx_http_gzip_filter_gzheader(ngx_http_request_t *r, ngx_http_gzip_ctx_t *ctx)
664 {
665 ngx_buf_t *b;
666 ngx_chain_t *cl;
667 static u_char gzheader[10] =
668 { 0x1f, 0x8b, Z_DEFLATED, 0, 0, 0, 0, 0, 0, 3 };
669
670 b = ngx_calloc_buf(r->pool);
671 if (b == NULL) {
672 return NGX_ERROR;
673 }
674
675 b->memory = 1;
676 b->pos = gzheader;
677 b->last = b->pos + 10;
678
679 cl = ngx_alloc_chain_link(r->pool);
680 if (cl == NULL) {
681 return NGX_ERROR;
682 }
683
684 cl->buf = b;
685 cl->next = ctx->out;
686 ctx->out = cl;
687
688 ctx->gzheader = 1;
689 629
690 return NGX_OK; 630 return NGX_OK;
691 } 631 }
692 632
693 633
741 if (ctx->in_buf->last_buf) { 681 if (ctx->in_buf->last_buf) {
742 ctx->flush = Z_FINISH; 682 ctx->flush = Z_FINISH;
743 683
744 } else if (ctx->in_buf->flush) { 684 } else if (ctx->in_buf->flush) {
745 ctx->flush = Z_SYNC_FLUSH; 685 ctx->flush = Z_SYNC_FLUSH;
746 } 686
747 687 } else if (ctx->zstream.avail_in == 0) {
748 if (ctx->zstream.avail_in) { 688 /* ctx->flush == Z_NO_FLUSH */
749
750 ctx->crc32 = crc32(ctx->crc32, ctx->zstream.next_in,
751 ctx->zstream.avail_in);
752
753 } else if (ctx->flush == Z_NO_FLUSH) {
754 return NGX_AGAIN; 689 return NGX_AGAIN;
755 } 690 }
756 691
757 return NGX_OK; 692 return NGX_OK;
758 } 693 }
930 865
931 static ngx_int_t 866 static ngx_int_t
932 ngx_http_gzip_filter_deflate_end(ngx_http_request_t *r, 867 ngx_http_gzip_filter_deflate_end(ngx_http_request_t *r,
933 ngx_http_gzip_ctx_t *ctx) 868 ngx_http_gzip_ctx_t *ctx)
934 { 869 {
935 int rc; 870 int rc;
936 ngx_buf_t *b; 871 ngx_chain_t *cl;
937 ngx_chain_t *cl;
938 struct gztrailer *trailer;
939 872
940 ctx->zin = ctx->zstream.total_in; 873 ctx->zin = ctx->zstream.total_in;
941 ctx->zout = 10 + ctx->zstream.total_out + 8; 874 ctx->zout = ctx->zstream.total_out;
942 875
943 rc = deflateEnd(&ctx->zstream); 876 rc = deflateEnd(&ctx->zstream);
944 877
945 if (rc != Z_OK) { 878 if (rc != Z_OK) {
946 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, 879 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0,
958 cl->buf = ctx->out_buf; 891 cl->buf = ctx->out_buf;
959 cl->next = NULL; 892 cl->next = NULL;
960 *ctx->last_out = cl; 893 *ctx->last_out = cl;
961 ctx->last_out = &cl->next; 894 ctx->last_out = &cl->next;
962 895
963 if (ctx->zstream.avail_out >= 8) { 896 ctx->out_buf->last_buf = 1;
964 trailer = (struct gztrailer *) ctx->out_buf->last;
965 ctx->out_buf->last += 8;
966 ctx->out_buf->last_buf = 1;
967
968 } else {
969 b = ngx_create_temp_buf(r->pool, 8);
970 if (b == NULL) {
971 return NGX_ERROR;
972 }
973
974 b->last_buf = 1;
975
976 cl = ngx_alloc_chain_link(r->pool);
977 if (cl == NULL) {
978 return NGX_ERROR;
979 }
980
981 cl->buf = b;
982 cl->next = NULL;
983 *ctx->last_out = cl;
984 ctx->last_out = &cl->next;
985 trailer = (struct gztrailer *) b->pos;
986 b->last += 8;
987 }
988
989 #if (NGX_HAVE_LITTLE_ENDIAN && NGX_HAVE_NONALIGNED)
990
991 trailer->crc32 = ctx->crc32;
992 trailer->zlen = ctx->zin;
993
994 #else
995
996 trailer->crc32[0] = (u_char) (ctx->crc32 & 0xff);
997 trailer->crc32[1] = (u_char) ((ctx->crc32 >> 8) & 0xff);
998 trailer->crc32[2] = (u_char) ((ctx->crc32 >> 16) & 0xff);
999 trailer->crc32[3] = (u_char) ((ctx->crc32 >> 24) & 0xff);
1000
1001 trailer->zlen[0] = (u_char) (ctx->zin & 0xff);
1002 trailer->zlen[1] = (u_char) ((ctx->zin >> 8) & 0xff);
1003 trailer->zlen[2] = (u_char) ((ctx->zin >> 16) & 0xff);
1004 trailer->zlen[3] = (u_char) ((ctx->zin >> 24) & 0xff);
1005
1006 #endif
1007 897
1008 ctx->zstream.avail_in = 0; 898 ctx->zstream.avail_in = 0;
1009 ctx->zstream.avail_out = 0; 899 ctx->zstream.avail_out = 0;
1010 900
1011 ctx->done = 1; 901 ctx->done = 1;