Mercurial > hg > nginx
changeset 6317:29f35e60840b
Slice filter.
Splits a request into subrequests, each providing a specific range of response.
The variable "$slice_range" must be used to set subrequest range and proper
cache key. The directive "slice" sets slice size.
The following example splits requests into 1-megabyte cacheable subrequests.
server {
listen 8000;
location / {
slice 1m;
proxy_cache cache;
proxy_cache_key $uri$is_args$args$slice_range;
proxy_set_header Range $slice_range;
proxy_cache_valid 200 206 1h;
proxy_pass http://127.0.0.1:9000;
}
}
| author | Roman Arutyunyan <arut@nginx.com> |
|---|---|
| date | Mon, 07 Dec 2015 16:30:48 +0300 |
| parents | f44de0d12143 |
| children | 3250a5783787 |
| files | auto/modules auto/options auto/sources src/http/modules/ngx_http_range_filter_module.c src/http/modules/ngx_http_slice_filter_module.c src/http/ngx_http_request.h |
| diffstat | 6 files changed, 567 insertions(+), 6 deletions(-) [+] |
line wrap: on
line diff
--- a/auto/modules +++ b/auto/modules @@ -73,6 +73,11 @@ if [ $HTTP_SSI = YES ]; then fi +if [ $HTTP_SLICE = YES ]; then + HTTP_POSTPONE=YES +fi + + if [ $HTTP_ADDITION = YES ]; then HTTP_POSTPONE=YES fi @@ -110,6 +115,7 @@ fi # ngx_http_copy_filter # ngx_http_range_body_filter # ngx_http_not_modified_filter +# ngx_http_slice_filter HTTP_FILTER_MODULES="$HTTP_WRITE_FILTER_MODULE \ $HTTP_HEADER_FILTER_MODULE \ @@ -179,6 +185,12 @@ if [ $HTTP_USERID = YES ]; then HTTP_SRCS="$HTTP_SRCS $HTTP_USERID_SRCS" fi +if [ $HTTP_SLICE = YES ]; then + HTTP_SRCS="$HTTP_SRCS $HTTP_SLICE_SRCS" +else + HTTP_SLICE_FILTER_MODULE="" +fi + if [ $HTTP_V2 = YES ]; then have=NGX_HTTP_V2 . auto/have @@ -461,7 +473,8 @@ if [ $HTTP = YES ]; then $HTTP_AUX_FILTER_MODULES \ $HTTP_COPY_FILTER_MODULE \ $HTTP_RANGE_BODY_FILTER_MODULE \ - $HTTP_NOT_MODIFIED_FILTER_MODULE" + $HTTP_NOT_MODIFIED_FILTER_MODULE \ + $HTTP_SLICE_FILTER_MODULE" NGX_ADDON_DEPS="$NGX_ADDON_DEPS \$(HTTP_DEPS)" fi
--- a/auto/options +++ b/auto/options @@ -71,6 +71,7 @@ HTTP_ACCESS=YES HTTP_AUTH_BASIC=YES HTTP_AUTH_REQUEST=NO HTTP_USERID=YES +HTTP_SLICE=NO HTTP_AUTOINDEX=YES HTTP_RANDOM_INDEX=NO HTTP_STATUS=NO @@ -226,6 +227,7 @@ do --with-http_random_index_module) HTTP_RANDOM_INDEX=YES ;; --with-http_secure_link_module) HTTP_SECURE_LINK=YES ;; --with-http_degradation_module) HTTP_DEGRADATION=YES ;; + --with-http_slice_module) HTTP_SLICE=YES ;; --without-http_charset_module) HTTP_CHARSET=NO ;; --without-http_gzip_module) HTTP_GZIP=NO ;; @@ -394,6 +396,7 @@ cat << END --with-http_random_index_module enable ngx_http_random_index_module --with-http_secure_link_module enable ngx_http_secure_link_module --with-http_degradation_module enable ngx_http_degradation_module + --with-http_slice_module enable ngx_http_slice_module --with-http_stub_status_module enable ngx_http_stub_status_module --without-http_charset_module disable ngx_http_charset_module
--- a/auto/sources +++ b/auto/sources @@ -360,6 +360,10 @@ HTTP_USERID_FILTER_MODULE=ngx_http_useri HTTP_USERID_SRCS=src/http/modules/ngx_http_userid_filter_module.c +HTTP_SLICE_FILTER_MODULE=ngx_http_slice_filter_module +HTTP_SLICE_SRCS=src/http/modules/ngx_http_slice_filter_module.c + + HTTP_REALIP_MODULE=ngx_http_realip_module HTTP_REALIP_SRCS=src/http/modules/ngx_http_realip_module.c
--- a/src/http/modules/ngx_http_range_filter_module.c +++ b/src/http/modules/ngx_http_range_filter_module.c @@ -154,7 +154,7 @@ ngx_http_range_header_filter(ngx_http_re if (r->http_version < NGX_HTTP_VERSION_10 || r->headers_out.status != NGX_HTTP_OK - || r != r->main + || (r != r->main && !r->subrequest_ranges) || r->headers_out.content_length_n == -1 || !r->allow_ranges) { @@ -222,6 +222,8 @@ parse: return NGX_ERROR; } + ctx->offset = r->headers_out.content_offset; + if (ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_range_t)) != NGX_OK) { @@ -273,10 +275,21 @@ static ngx_int_t ngx_http_range_parse(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx, ngx_uint_t ranges) { - u_char *p; - off_t start, end, size, content_length, cutoff, cutlim; - ngx_uint_t suffix; - ngx_http_range_t *range; + u_char *p; + off_t start, end, size, content_length, cutoff, + cutlim; + ngx_uint_t suffix; + ngx_http_range_t *range; + ngx_http_range_filter_ctx_t *mctx; + + if (r != r->main) { + mctx = ngx_http_get_module_ctx(r->main, + ngx_http_range_body_filter_module); + if (mctx) { + ctx->ranges = mctx->ranges; + return NGX_OK; + } + } p = r->headers_in.range->value.data + 6; size = 0; @@ -395,6 +408,10 @@ ngx_http_range_singlepart_header(ngx_htt ngx_table_elt_t *content_range; ngx_http_range_t *range; + if (r != r->main) { + return ngx_http_next_header_filter(r); + } + content_range = ngx_list_push(&r->headers_out.headers); if (content_range == NULL) { return NGX_ERROR; @@ -422,6 +439,7 @@ ngx_http_range_singlepart_header(ngx_htt - content_range->value.data; r->headers_out.content_length_n = range->end - range->start; + r->headers_out.content_offset = range->start; if (r->headers_out.content_length) { r->headers_out.content_length->hash = 0;
new file mode 100644 --- /dev/null +++ b/src/http/modules/ngx_http_slice_filter_module.c @@ -0,0 +1,521 @@ + +/* + * Copyright (C) Roman Arutyunyan + * Copyright (C) Nginx, Inc. + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_http.h> + + +typedef struct { + size_t size; +} ngx_http_slice_loc_conf_t; + + +typedef struct { + off_t start; + off_t end; + ngx_str_t range; + ngx_str_t etag; + ngx_uint_t last; /* unsigned last:1; */ +} ngx_http_slice_ctx_t; + + +typedef struct { + off_t start; + off_t end; + off_t complete_length; +} ngx_http_slice_content_range_t; + + +static ngx_int_t ngx_http_slice_header_filter(ngx_http_request_t *r); +static ngx_int_t ngx_http_slice_body_filter(ngx_http_request_t *r, + ngx_chain_t *in); +static ngx_int_t ngx_http_slice_parse_content_range(ngx_http_request_t *r, + ngx_http_slice_content_range_t *cr); +static ngx_int_t ngx_http_slice_range_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); +static off_t ngx_http_slice_get_start(ngx_http_request_t *r); +static void *ngx_http_slice_create_loc_conf(ngx_conf_t *cf); +static char *ngx_http_slice_merge_loc_conf(ngx_conf_t *cf, void *parent, + void *child); +static ngx_int_t ngx_http_slice_add_variables(ngx_conf_t *cf); +static ngx_int_t ngx_http_slice_init(ngx_conf_t *cf); + + +static ngx_command_t ngx_http_slice_filter_commands[] = { + + { ngx_string("slice"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_slice_loc_conf_t, size), + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_slice_filter_module_ctx = { + ngx_http_slice_add_variables, /* preconfiguration */ + ngx_http_slice_init, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + ngx_http_slice_create_loc_conf, /* create location configuration */ + ngx_http_slice_merge_loc_conf /* merge location configuration */ +}; + + +ngx_module_t ngx_http_slice_filter_module = { + NGX_MODULE_V1, + &ngx_http_slice_filter_module_ctx, /* module context */ + ngx_http_slice_filter_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_str_t ngx_http_slice_range_name = ngx_string("slice_range"); + +static ngx_http_output_header_filter_pt ngx_http_next_header_filter; +static ngx_http_output_body_filter_pt ngx_http_next_body_filter; + + +static ngx_int_t +ngx_http_slice_header_filter(ngx_http_request_t *r) +{ + off_t end; + ngx_int_t rc; + ngx_table_elt_t *h; + ngx_http_slice_ctx_t *ctx; + ngx_http_slice_loc_conf_t *slcf; + ngx_http_slice_content_range_t cr; + + ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module); + if (ctx == NULL) { + return ngx_http_next_header_filter(r); + } + + if (r->headers_out.status != NGX_HTTP_PARTIAL_CONTENT) { + if (r == r->main) { + ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module); + return ngx_http_next_header_filter(r); + } + + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "unexpected status code %ui in slice response", + r->headers_out.status); + return NGX_ERROR; + } + + h = r->headers_out.etag; + + if (ctx->etag.len) { + if (h == NULL + || h->value.len != ctx->etag.len + || ngx_strncmp(h->value.data, ctx->etag.data, ctx->etag.len) + != 0) + { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "etag mismatch in slice response"); + return NGX_ERROR; + } + } + + if (h) { + ctx->etag = h->value; + } + + if (ngx_http_slice_parse_content_range(r, &cr) != NGX_OK) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "invalid range in slice response"); + return NGX_ERROR; + } + + if (cr.complete_length == -1) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "no complete length in slice response"); + return NGX_ERROR; + } + + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http slice response range: %O-%O/%O", + cr.start, cr.end, cr.complete_length); + + slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module); + + end = ngx_min(cr.start + (off_t) slcf->size, cr.complete_length); + + if (cr.start != ctx->start || cr.end != end) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "unexpected range in slice response: %O-%O", + cr.start, cr.end); + return NGX_ERROR; + } + + ctx->start = end; + + r->headers_out.status = NGX_HTTP_OK; + r->headers_out.status_line.len = 0; + r->headers_out.content_length_n = cr.complete_length; + r->headers_out.content_offset = cr.start; + r->headers_out.content_range->hash = 0; + r->headers_out.content_range = NULL; + + r->allow_ranges = 1; + r->subrequest_ranges = 1; + r->single_range = 1; + + rc = ngx_http_next_header_filter(r); + + if (r != r->main) { + return rc; + } + + if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) { + if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) { + ctx->start = slcf->size + * (r->headers_out.content_offset / slcf->size); + } + + ctx->end = r->headers_out.content_offset + + r->headers_out.content_length_n; + + } else { + ctx->end = cr.complete_length; + } + + return rc; +} + + +static ngx_int_t +ngx_http_slice_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + ngx_int_t rc; + ngx_chain_t *cl; + ngx_http_request_t *sr; + ngx_http_slice_ctx_t *ctx; + ngx_http_slice_loc_conf_t *slcf; + + ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module); + + if (ctx == NULL || r != r->main) { + return ngx_http_next_body_filter(r, in); + } + + for (cl = in; cl; cl = cl->next) { + if (cl->buf->last_buf) { + cl->buf->last_buf = 0; + cl->buf->sync = 1; + ctx->last = 1; + } + } + + rc = ngx_http_next_body_filter(r, in); + + if (rc == NGX_ERROR || !ctx->last) { + return rc; + } + + if (ctx->start >= ctx->end) { + ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module); + ngx_http_send_special(r, NGX_HTTP_LAST); + return rc; + } + + if (ngx_http_subrequest(r, &r->uri, &r->args, &sr, NULL, 0) != NGX_OK) { + return NGX_ERROR; + } + + ngx_http_set_ctx(sr, ctx, ngx_http_slice_filter_module); + + slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module); + + ctx->range.len = ngx_sprintf(ctx->range.data, "bytes=%O-%O", ctx->start, + ctx->start + (off_t) slcf->size - 1) + - ctx->range.data; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http slice subrequest: \"%V\"", &ctx->range); + + return rc; +} + + +static ngx_int_t +ngx_http_slice_parse_content_range(ngx_http_request_t *r, + ngx_http_slice_content_range_t *cr) +{ + off_t start, end, complete_length, cutoff, cutlim; + u_char *p; + ngx_table_elt_t *h; + + h = r->headers_out.content_range; + + if (h == NULL + || h->value.len < 7 + || ngx_strncmp(h->value.data, "bytes ", 6) != 0) + { + return NGX_ERROR; + } + + p = h->value.data + 6; + + cutoff = NGX_MAX_OFF_T_VALUE / 10; + cutlim = NGX_MAX_OFF_T_VALUE % 10; + + start = 0; + end = 0; + complete_length = 0; + + while (*p == ' ') { p++; } + + if (*p < '0' || *p > '9') { + return NGX_ERROR; + } + + while (*p >= '0' && *p <= '9') { + if (start >= cutoff && (start > cutoff || *p - '0' > cutlim)) { + return NGX_ERROR; + } + + start = start * 10 + *p++ - '0'; + } + + while (*p == ' ') { p++; } + + if (*p++ != '-') { + return NGX_ERROR; + } + + while (*p == ' ') { p++; } + + if (*p < '0' || *p > '9') { + return NGX_ERROR; + } + + while (*p >= '0' && *p <= '9') { + if (end >= cutoff && (end > cutoff || *p - '0' > cutlim)) { + return NGX_ERROR; + } + + end = end * 10 + *p++ - '0'; + } + + end++; + + while (*p == ' ') { p++; } + + if (*p++ != '/') { + return NGX_ERROR; + } + + while (*p == ' ') { p++; } + + if (*p != '*') { + if (*p < '0' || *p > '9') { + return NGX_ERROR; + } + + while (*p >= '0' && *p <= '9') { + if (complete_length >= cutoff + && (complete_length > cutoff || *p - '0' > cutlim)) + { + return NGX_ERROR; + } + + complete_length = complete_length * 10 + *p++ - '0'; + } + + } else { + complete_length = -1; + p++; + } + + while (*p == ' ') { p++; } + + if (*p != '\0') { + return NGX_ERROR; + } + + cr->start = start; + cr->end = end; + cr->complete_length = complete_length; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_slice_range_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + u_char *p; + ngx_http_slice_ctx_t *ctx; + ngx_http_slice_loc_conf_t *slcf; + + ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module); + + if (ctx == NULL) { + if (r != r->main || r->headers_out.status) { + v->not_found = 1; + return NGX_OK; + } + + slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module); + + if (slcf->size == 0) { + v->not_found = 1; + return NGX_OK; + } + + ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_slice_ctx_t)); + if (ctx == NULL) { + return NGX_ERROR; + } + + ngx_http_set_ctx(r, ctx, ngx_http_slice_filter_module); + + p = ngx_pnalloc(r->pool, sizeof("bytes=-") - 1 + 2 * NGX_OFF_T_LEN); + if (p == NULL) { + return NGX_ERROR; + } + + ctx->start = slcf->size * (ngx_http_slice_get_start(r) / slcf->size); + + ctx->range.data = p; + ctx->range.len = ngx_sprintf(p, "bytes=%O-%O", ctx->start, + ctx->start + (off_t) slcf->size - 1) + - p; + } + + v->data = ctx->range.data; + v->valid = 1; + v->not_found = 0; + v->no_cacheable = 1; + v->len = ctx->range.len; + + return NGX_OK; +} + + +static off_t +ngx_http_slice_get_start(ngx_http_request_t *r) +{ + off_t start, cutoff, cutlim; + u_char *p; + ngx_table_elt_t *h; + + if (r->headers_in.if_range) { + return 0; + } + + h = r->headers_in.range; + + if (h == NULL + || h->value.len < 7 + || ngx_strncasecmp(h->value.data, (u_char *) "bytes=", 6) != 0) + { + return 0; + } + + p = h->value.data + 6; + + if (ngx_strchr(p, ',')) { + return 0; + } + + while (*p == ' ') { p++; } + + if (*p == '-') { + return 0; + } + + cutoff = NGX_MAX_OFF_T_VALUE / 10; + cutlim = NGX_MAX_OFF_T_VALUE % 10; + + start = 0; + + while (*p >= '0' && *p <= '9') { + if (start >= cutoff && (start > cutoff || *p - '0' > cutlim)) { + return 0; + } + + start = start * 10 + *p++ - '0'; + } + + return start; +} + + +static void * +ngx_http_slice_create_loc_conf(ngx_conf_t *cf) +{ + ngx_http_slice_loc_conf_t *slcf; + + slcf = ngx_palloc(cf->pool, sizeof(ngx_http_slice_loc_conf_t)); + if (slcf == NULL) { + return NULL; + } + + slcf->size = NGX_CONF_UNSET_SIZE; + + return slcf; +} + + +static char * +ngx_http_slice_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) +{ + ngx_http_slice_loc_conf_t *prev = parent; + ngx_http_slice_loc_conf_t *conf = child; + + ngx_conf_merge_size_value(conf->size, prev->size, 0); + + return NGX_CONF_OK; +} + + +static ngx_int_t +ngx_http_slice_add_variables(ngx_conf_t *cf) +{ + ngx_http_variable_t *var; + + var = ngx_http_add_variable(cf, &ngx_http_slice_range_name, 0); + if (var == NULL) { + return NGX_ERROR; + } + + var->get_handler = ngx_http_slice_range_variable; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_slice_init(ngx_conf_t *cf) +{ + ngx_http_next_header_filter = ngx_http_top_header_filter; + ngx_http_top_header_filter = ngx_http_slice_header_filter; + + ngx_http_next_body_filter = ngx_http_top_body_filter; + ngx_http_top_body_filter = ngx_http_slice_body_filter; + + return NGX_OK; +}
--- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -271,6 +271,7 @@ typedef struct { ngx_array_t cache_control; off_t content_length_n; + off_t content_offset; time_t date_time; time_t last_modified_time; } ngx_http_headers_out_t; @@ -530,6 +531,7 @@ struct ngx_http_request_s { unsigned filter_need_in_memory:1; unsigned filter_need_temporary:1; unsigned allow_ranges:1; + unsigned subrequest_ranges:1; unsigned single_range:1; unsigned disable_not_modified:1;