Mercurial > hg > nginx-quic
comparison src/http/modules/ngx_http_gzip_filter_module.c @ 7534:ac5a741d39cf
Gzip: use zlib to write header and trailer.
When nginx is used with zlib patched with [1], which provides
integration with the future IBM Z hardware deflate acceleration, it ends
up computing CRC32 twice: one time in hardware, which always does this,
and one time in software by explicitly calling crc32().
crc32() calls were added in changesets 133:b27548f540ad ("nginx-0.0.1-
2003-09-24-23:51:12 import") and 134:d57c6835225c ("nginx-0.0.1-
2003-09-26-09:45:21 import") as part of gzip wrapping feature - back
then zlib did not support it.
However, since then gzip wrapping was implemented in zlib v1.2.0.4,
and it's already being used by nginx for log compression.
This patch replaces hand-written gzip wrapping with the one provided by
zlib. It simplifies the code, and makes it avoid computing CRC32 twice
when using hardware acceleration.
[1] https://github.com/madler/zlib/pull/410
author | Ilya Leoshkevich <iii@linux.ibm.com> |
---|---|
date | Fri, 12 Jul 2019 12:43:08 +0200 |
parents | 29e9571b1989 |
children | 0ef2bc0ec9c9 |
comparison
equal
deleted
inserted
replaced
7533:5f642712e7ad | 7534:ac5a741d39cf |
---|---|
53 | 53 |
54 unsigned flush:4; | 54 unsigned flush:4; |
55 unsigned redo:1; | 55 unsigned redo:1; |
56 unsigned done:1; | 56 unsigned done:1; |
57 unsigned nomem:1; | 57 unsigned nomem:1; |
58 unsigned gzheader:1; | |
59 unsigned buffering:1; | 58 unsigned buffering:1; |
60 unsigned intel:1; | 59 unsigned intel:1; |
61 | 60 |
62 size_t zin; | 61 size_t zin; |
63 size_t zout; | 62 size_t zout; |
64 | 63 |
65 uint32_t crc32; | |
66 z_stream zstream; | 64 z_stream zstream; |
67 ngx_http_request_t *request; | 65 ngx_http_request_t *request; |
68 } ngx_http_gzip_ctx_t; | 66 } ngx_http_gzip_ctx_t; |
69 | |
70 | |
71 #if (NGX_HAVE_LITTLE_ENDIAN && NGX_HAVE_NONALIGNED) | |
72 | |
73 struct gztrailer { | |
74 uint32_t crc32; | |
75 uint32_t zlen; | |
76 }; | |
77 | |
78 #else /* NGX_HAVE_BIG_ENDIAN || !NGX_HAVE_NONALIGNED */ | |
79 | |
80 struct gztrailer { | |
81 u_char crc32[4]; | |
82 u_char zlen[4]; | |
83 }; | |
84 | |
85 #endif | |
86 | 67 |
87 | 68 |
88 static void ngx_http_gzip_filter_memory(ngx_http_request_t *r, | 69 static void ngx_http_gzip_filter_memory(ngx_http_request_t *r, |
89 ngx_http_gzip_ctx_t *ctx); | 70 ngx_http_gzip_ctx_t *ctx); |
90 static ngx_int_t ngx_http_gzip_filter_buffer(ngx_http_gzip_ctx_t *ctx, | 71 static ngx_int_t ngx_http_gzip_filter_buffer(ngx_http_gzip_ctx_t *ctx, |
91 ngx_chain_t *in); | 72 ngx_chain_t *in); |
92 static ngx_int_t ngx_http_gzip_filter_deflate_start(ngx_http_request_t *r, | 73 static ngx_int_t ngx_http_gzip_filter_deflate_start(ngx_http_request_t *r, |
93 ngx_http_gzip_ctx_t *ctx); | |
94 static ngx_int_t ngx_http_gzip_filter_gzheader(ngx_http_request_t *r, | |
95 ngx_http_gzip_ctx_t *ctx); | 74 ngx_http_gzip_ctx_t *ctx); |
96 static ngx_int_t ngx_http_gzip_filter_add_data(ngx_http_request_t *r, | 75 static ngx_int_t ngx_http_gzip_filter_add_data(ngx_http_request_t *r, |
97 ngx_http_gzip_ctx_t *ctx); | 76 ngx_http_gzip_ctx_t *ctx); |
98 static ngx_int_t ngx_http_gzip_filter_get_buf(ngx_http_request_t *r, | 77 static ngx_int_t ngx_http_gzip_filter_get_buf(ngx_http_request_t *r, |
99 ngx_http_gzip_ctx_t *ctx); | 78 ngx_http_gzip_ctx_t *ctx); |
444 ngx_http_gzip_filter_free_copy_buf(r, ctx); | 423 ngx_http_gzip_filter_free_copy_buf(r, ctx); |
445 | 424 |
446 return ctx->busy ? NGX_AGAIN : NGX_OK; | 425 return ctx->busy ? NGX_AGAIN : NGX_OK; |
447 } | 426 } |
448 | 427 |
449 if (!ctx->gzheader) { | |
450 if (ngx_http_gzip_filter_gzheader(r, ctx) != NGX_OK) { | |
451 goto failed; | |
452 } | |
453 } | |
454 | |
455 rc = ngx_http_next_body_filter(r, ctx->out); | 428 rc = ngx_http_next_body_filter(r, ctx->out); |
456 | 429 |
457 if (rc == NGX_ERROR) { | 430 if (rc == NGX_ERROR) { |
458 goto failed; | 431 goto failed; |
459 } | 432 } |
641 ctx->zstream.zalloc = ngx_http_gzip_filter_alloc; | 614 ctx->zstream.zalloc = ngx_http_gzip_filter_alloc; |
642 ctx->zstream.zfree = ngx_http_gzip_filter_free; | 615 ctx->zstream.zfree = ngx_http_gzip_filter_free; |
643 ctx->zstream.opaque = ctx; | 616 ctx->zstream.opaque = ctx; |
644 | 617 |
645 rc = deflateInit2(&ctx->zstream, (int) conf->level, Z_DEFLATED, | 618 rc = deflateInit2(&ctx->zstream, (int) conf->level, Z_DEFLATED, |
646 - ctx->wbits, ctx->memlevel, Z_DEFAULT_STRATEGY); | 619 ctx->wbits + 16, ctx->memlevel, Z_DEFAULT_STRATEGY); |
647 | 620 |
648 if (rc != Z_OK) { | 621 if (rc != Z_OK) { |
649 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, | 622 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, |
650 "deflateInit2() failed: %d", rc); | 623 "deflateInit2() failed: %d", rc); |
651 return NGX_ERROR; | 624 return NGX_ERROR; |
652 } | 625 } |
653 | 626 |
654 ctx->last_out = &ctx->out; | 627 ctx->last_out = &ctx->out; |
655 ctx->crc32 = crc32(0L, Z_NULL, 0); | |
656 ctx->flush = Z_NO_FLUSH; | 628 ctx->flush = Z_NO_FLUSH; |
657 | |
658 return NGX_OK; | |
659 } | |
660 | |
661 | |
662 static ngx_int_t | |
663 ngx_http_gzip_filter_gzheader(ngx_http_request_t *r, ngx_http_gzip_ctx_t *ctx) | |
664 { | |
665 ngx_buf_t *b; | |
666 ngx_chain_t *cl; | |
667 static u_char gzheader[10] = | |
668 { 0x1f, 0x8b, Z_DEFLATED, 0, 0, 0, 0, 0, 0, 3 }; | |
669 | |
670 b = ngx_calloc_buf(r->pool); | |
671 if (b == NULL) { | |
672 return NGX_ERROR; | |
673 } | |
674 | |
675 b->memory = 1; | |
676 b->pos = gzheader; | |
677 b->last = b->pos + 10; | |
678 | |
679 cl = ngx_alloc_chain_link(r->pool); | |
680 if (cl == NULL) { | |
681 return NGX_ERROR; | |
682 } | |
683 | |
684 cl->buf = b; | |
685 cl->next = ctx->out; | |
686 ctx->out = cl; | |
687 | |
688 ctx->gzheader = 1; | |
689 | 629 |
690 return NGX_OK; | 630 return NGX_OK; |
691 } | 631 } |
692 | 632 |
693 | 633 |
741 if (ctx->in_buf->last_buf) { | 681 if (ctx->in_buf->last_buf) { |
742 ctx->flush = Z_FINISH; | 682 ctx->flush = Z_FINISH; |
743 | 683 |
744 } else if (ctx->in_buf->flush) { | 684 } else if (ctx->in_buf->flush) { |
745 ctx->flush = Z_SYNC_FLUSH; | 685 ctx->flush = Z_SYNC_FLUSH; |
746 } | 686 |
747 | 687 } else if (ctx->zstream.avail_in == 0) { |
748 if (ctx->zstream.avail_in) { | 688 /* ctx->flush == Z_NO_FLUSH */ |
749 | |
750 ctx->crc32 = crc32(ctx->crc32, ctx->zstream.next_in, | |
751 ctx->zstream.avail_in); | |
752 | |
753 } else if (ctx->flush == Z_NO_FLUSH) { | |
754 return NGX_AGAIN; | 689 return NGX_AGAIN; |
755 } | 690 } |
756 | 691 |
757 return NGX_OK; | 692 return NGX_OK; |
758 } | 693 } |
930 | 865 |
931 static ngx_int_t | 866 static ngx_int_t |
932 ngx_http_gzip_filter_deflate_end(ngx_http_request_t *r, | 867 ngx_http_gzip_filter_deflate_end(ngx_http_request_t *r, |
933 ngx_http_gzip_ctx_t *ctx) | 868 ngx_http_gzip_ctx_t *ctx) |
934 { | 869 { |
935 int rc; | 870 int rc; |
936 ngx_buf_t *b; | 871 ngx_chain_t *cl; |
937 ngx_chain_t *cl; | |
938 struct gztrailer *trailer; | |
939 | 872 |
940 ctx->zin = ctx->zstream.total_in; | 873 ctx->zin = ctx->zstream.total_in; |
941 ctx->zout = 10 + ctx->zstream.total_out + 8; | 874 ctx->zout = ctx->zstream.total_out; |
942 | 875 |
943 rc = deflateEnd(&ctx->zstream); | 876 rc = deflateEnd(&ctx->zstream); |
944 | 877 |
945 if (rc != Z_OK) { | 878 if (rc != Z_OK) { |
946 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, | 879 ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, |
958 cl->buf = ctx->out_buf; | 891 cl->buf = ctx->out_buf; |
959 cl->next = NULL; | 892 cl->next = NULL; |
960 *ctx->last_out = cl; | 893 *ctx->last_out = cl; |
961 ctx->last_out = &cl->next; | 894 ctx->last_out = &cl->next; |
962 | 895 |
963 if (ctx->zstream.avail_out >= 8) { | 896 ctx->out_buf->last_buf = 1; |
964 trailer = (struct gztrailer *) ctx->out_buf->last; | |
965 ctx->out_buf->last += 8; | |
966 ctx->out_buf->last_buf = 1; | |
967 | |
968 } else { | |
969 b = ngx_create_temp_buf(r->pool, 8); | |
970 if (b == NULL) { | |
971 return NGX_ERROR; | |
972 } | |
973 | |
974 b->last_buf = 1; | |
975 | |
976 cl = ngx_alloc_chain_link(r->pool); | |
977 if (cl == NULL) { | |
978 return NGX_ERROR; | |
979 } | |
980 | |
981 cl->buf = b; | |
982 cl->next = NULL; | |
983 *ctx->last_out = cl; | |
984 ctx->last_out = &cl->next; | |
985 trailer = (struct gztrailer *) b->pos; | |
986 b->last += 8; | |
987 } | |
988 | |
989 #if (NGX_HAVE_LITTLE_ENDIAN && NGX_HAVE_NONALIGNED) | |
990 | |
991 trailer->crc32 = ctx->crc32; | |
992 trailer->zlen = ctx->zin; | |
993 | |
994 #else | |
995 | |
996 trailer->crc32[0] = (u_char) (ctx->crc32 & 0xff); | |
997 trailer->crc32[1] = (u_char) ((ctx->crc32 >> 8) & 0xff); | |
998 trailer->crc32[2] = (u_char) ((ctx->crc32 >> 16) & 0xff); | |
999 trailer->crc32[3] = (u_char) ((ctx->crc32 >> 24) & 0xff); | |
1000 | |
1001 trailer->zlen[0] = (u_char) (ctx->zin & 0xff); | |
1002 trailer->zlen[1] = (u_char) ((ctx->zin >> 8) & 0xff); | |
1003 trailer->zlen[2] = (u_char) ((ctx->zin >> 16) & 0xff); | |
1004 trailer->zlen[3] = (u_char) ((ctx->zin >> 24) & 0xff); | |
1005 | |
1006 #endif | |
1007 | 897 |
1008 ctx->zstream.avail_in = 0; | 898 ctx->zstream.avail_in = 0; |
1009 ctx->zstream.avail_out = 0; | 899 ctx->zstream.avail_out = 0; |
1010 | 900 |
1011 ctx->done = 1; | 901 ctx->done = 1; |