# HG changeset patch
# User Roman Arutyunyan <arut@nginx.com>
# Date 1519826218 -10800
# Node ID 20f139e9ffa84f1a1db6039bbbb547cd35fc4534
# Parent  d0d32b33167d493722fb986e642a28cab21b0084
Generic subrequests in memory.

Previously, only the upstream response body could be accessed with the
NGX_HTTP_SUBREQUEST_IN_MEMORY feature.  Now any response body from a subrequest
can be saved in a memory buffer.  It is available as a single buffer in r->out
and the buffer size is configured by the subrequest_output_buffer_size
directive.

Upstream, proxy and fastcgi code used to handle the old-style feature is
removed.

diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c
--- a/src/http/modules/ngx_http_fastcgi_module.c
+++ b/src/http/modules/ngx_http_fastcgi_module.c
@@ -2512,36 +2512,6 @@ ngx_http_fastcgi_non_buffered_filter(voi
         break;
     }
 
-    /* provide continuous buffer for subrequests in memory */
-
-    if (r->subrequest_in_memory) {
-
-        cl = u->out_bufs;
-
-        if (cl) {
-            buf->pos = cl->buf->pos;
-        }
-
-        buf->last = buf->pos;
-
-        for (cl = u->out_bufs; cl; cl = cl->next) {
-            ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                           "http fastcgi in memory %p-%p %O",
-                           cl->buf->pos, cl->buf->last, ngx_buf_size(cl->buf));
-
-            if (buf->last == cl->buf->pos) {
-                buf->last = cl->buf->last;
-                continue;
-            }
-
-            buf->last = ngx_movemem(buf->last, cl->buf->pos,
-                                    cl->buf->last - cl->buf->pos);
-
-            cl->buf->pos = buf->last - (cl->buf->last - cl->buf->pos);
-            cl->buf->last = buf->last;
-        }
-    }
-
     return NGX_OK;
 }
 
diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -2321,36 +2321,6 @@ ngx_http_proxy_non_buffered_chunked_filt
         return NGX_ERROR;
     }
 
-    /* provide continuous buffer for subrequests in memory */
-
-    if (r->subrequest_in_memory) {
-
-        cl = u->out_bufs;
-
-        if (cl) {
-            buf->pos = cl->buf->pos;
-        }
-
-        buf->last = buf->pos;
-
-        for (cl = u->out_bufs; cl; cl = cl->next) {
-            ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
-                           "http proxy in memory %p-%p %O",
-                           cl->buf->pos, cl->buf->last, ngx_buf_size(cl->buf));
-
-            if (buf->last == cl->buf->pos) {
-                buf->last = cl->buf->last;
-                continue;
-            }
-
-            buf->last = ngx_movemem(buf->last, cl->buf->pos,
-                                    cl->buf->last - cl->buf->pos);
-
-            cl->buf->pos = buf->last - (cl->buf->last - cl->buf->pos);
-            cl->buf->last = buf->last;
-        }
-    }
-
     return NGX_OK;
 }
 
diff --git a/src/http/modules/ngx_http_ssi_filter_module.c b/src/http/modules/ngx_http_ssi_filter_module.c
--- a/src/http/modules/ngx_http_ssi_filter_module.c
+++ b/src/http/modules/ngx_http_ssi_filter_module.c
@@ -2231,9 +2231,11 @@ ngx_http_ssi_set_variable(ngx_http_reque
 {
     ngx_str_t  *value = data;
 
-    if (r->upstream) {
-        value->len = r->upstream->buffer.last - r->upstream->buffer.pos;
-        value->data = r->upstream->buffer.pos;
+    if (r->headers_out.status < NGX_HTTP_SPECIAL_RESPONSE
+        && r->out && r->out->buf)
+    {
+        value->len = r->out->buf->last - r->out->buf->pos;
+        value->data = r->out->buf->pos;
     }
 
     return rc;
diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c
--- a/src/http/ngx_http_core_module.c
+++ b/src/http/ngx_http_core_module.c
@@ -399,6 +399,13 @@ static ngx_command_t  ngx_http_core_comm
       offsetof(ngx_http_core_loc_conf_t, sendfile_max_chunk),
       NULL },
 
+    { ngx_string("subrequest_output_buffer_size"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_size_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_core_loc_conf_t, subrequest_output_buffer_size),
+      NULL },
+
     { ngx_string("aio"),
       NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
       ngx_http_core_set_aio,
@@ -2237,6 +2244,12 @@ ngx_http_subrequest(ngx_http_request_t *
         return NGX_ERROR;
     }
 
+    if (r->subrequest_in_memory) {
+        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                      "nested in-memory subrequest \"%V\"", uri);
+        return NGX_ERROR;
+    }
+
     sr = ngx_pcalloc(r->pool, sizeof(ngx_http_request_t));
     if (sr == NULL) {
         return NGX_ERROR;
@@ -2318,6 +2331,10 @@ ngx_http_subrequest(ngx_http_request_t *
 
     sr->log_handler = r->log_handler;
 
+    if (sr->subrequest_in_memory) {
+        sr->filter_need_in_memory = 1;
+    }
+
     if (!sr->background) {
         if (c->data == r && r->postponed == NULL) {
             c->data = sr;
@@ -3356,6 +3373,7 @@ ngx_http_core_create_loc_conf(ngx_conf_t
     clcf->internal = NGX_CONF_UNSET;
     clcf->sendfile = NGX_CONF_UNSET;
     clcf->sendfile_max_chunk = NGX_CONF_UNSET_SIZE;
+    clcf->subrequest_output_buffer_size = NGX_CONF_UNSET_SIZE;
     clcf->aio = NGX_CONF_UNSET;
     clcf->aio_write = NGX_CONF_UNSET;
 #if (NGX_THREADS)
@@ -3578,6 +3596,9 @@ ngx_http_core_merge_loc_conf(ngx_conf_t 
     ngx_conf_merge_value(conf->sendfile, prev->sendfile, 0);
     ngx_conf_merge_size_value(conf->sendfile_max_chunk,
                               prev->sendfile_max_chunk, 0);
+    ngx_conf_merge_size_value(conf->subrequest_output_buffer_size,
+                              prev->subrequest_output_buffer_size,
+                              (size_t) ngx_pagesize);
     ngx_conf_merge_value(conf->aio, prev->aio, NGX_HTTP_AIO_OFF);
     ngx_conf_merge_value(conf->aio_write, prev->aio_write, 0);
 #if (NGX_THREADS)
diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h
--- a/src/http/ngx_http_core_module.h
+++ b/src/http/ngx_http_core_module.h
@@ -351,6 +351,8 @@ struct ngx_http_core_loc_conf_s {
     size_t        limit_rate_after;        /* limit_rate_after */
     size_t        sendfile_max_chunk;      /* sendfile_max_chunk */
     size_t        read_ahead;              /* read_ahead */
+    size_t        subrequest_output_buffer_size;
+                                           /* subrequest_output_buffer_size */
 
     ngx_msec_t    client_body_timeout;     /* client_body_timeout */
     ngx_msec_t    send_timeout;            /* send_timeout */
diff --git a/src/http/ngx_http_postpone_filter_module.c b/src/http/ngx_http_postpone_filter_module.c
--- a/src/http/ngx_http_postpone_filter_module.c
+++ b/src/http/ngx_http_postpone_filter_module.c
@@ -12,6 +12,8 @@
 
 static ngx_int_t ngx_http_postpone_filter_add(ngx_http_request_t *r,
     ngx_chain_t *in);
+static ngx_int_t ngx_http_postpone_filter_in_memory(ngx_http_request_t *r,
+    ngx_chain_t *in);
 static ngx_int_t ngx_http_postpone_filter_init(ngx_conf_t *cf);
 
 
@@ -60,6 +62,10 @@ ngx_http_postpone_filter(ngx_http_reques
     ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http postpone filter \"%V?%V\" %p", &r->uri, &r->args, in);
 
+    if (r->subrequest_in_memory) {
+        return ngx_http_postpone_filter_in_memory(r, in);
+    }
+
     if (r != c->data) {
 
         if (in) {
@@ -172,6 +178,78 @@ found:
 
 
 static ngx_int_t
+ngx_http_postpone_filter_in_memory(ngx_http_request_t *r, ngx_chain_t *in)
+{
+    size_t                     len;
+    ngx_buf_t                 *b;
+    ngx_connection_t          *c;
+    ngx_http_core_loc_conf_t  *clcf;
+
+    c = r->connection;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                   "http postpone filter in memory");
+
+    if (r->out == NULL) {
+        r->out = ngx_alloc_chain_link(r->pool);
+        if (r->out == NULL) {
+            return NGX_ERROR;
+        }
+
+        clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+        if (r->headers_out.content_length_n != -1) {
+            len = r->headers_out.content_length_n;
+
+            if (len > clcf->subrequest_output_buffer_size) {
+                ngx_log_error(NGX_LOG_ERR, c->log, 0,
+                              "too big subrequest response: %uz", len);
+                return NGX_ERROR;
+            }
+
+        } else {
+            len = clcf->subrequest_output_buffer_size;
+        }
+
+        b = ngx_create_temp_buf(r->pool, len);
+        if (b == NULL) {
+            return NGX_ERROR;
+        }
+
+        b->last_buf = 1;
+
+        r->out->buf = b;
+        r->out->next = NULL;
+    }
+
+    b = r->out->buf;
+
+    for ( /* void */ ; in; in = in->next) {
+
+        if (ngx_buf_special(in->buf)) {
+            continue;
+        }
+
+        len = in->buf->last - in->buf->pos;
+
+        if (len > (size_t) (b->end - b->last)) {
+            ngx_log_error(NGX_LOG_ERR, c->log, 0,
+                          "too big subrequest response");
+            return NGX_ERROR;
+        }
+
+        ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                       "http postpone filter in memory %uz bytes", len);
+
+        b->last = ngx_cpymem(b->last, in->buf->pos, len);
+        in->buf->pos = in->buf->last;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
 ngx_http_postpone_filter_init(ngx_conf_t *cf)
 {
     ngx_http_next_body_filter = ngx_http_top_body_filter;
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -55,8 +55,6 @@ static ngx_int_t ngx_http_upstream_inter
 static ngx_int_t ngx_http_upstream_test_connect(ngx_connection_t *c);
 static ngx_int_t ngx_http_upstream_process_headers(ngx_http_request_t *r,
     ngx_http_upstream_t *u);
-static void ngx_http_upstream_process_body_in_memory(ngx_http_request_t *r,
-    ngx_http_upstream_t *u);
 static void ngx_http_upstream_send_response(ngx_http_request_t *r,
     ngx_http_upstream_t *u);
 static void ngx_http_upstream_upgrade(ngx_http_request_t *r,
@@ -2335,45 +2333,7 @@ ngx_http_upstream_process_header(ngx_htt
         return;
     }
 
-    if (!r->subrequest_in_memory) {
-        ngx_http_upstream_send_response(r, u);
-        return;
-    }
-
-    /* subrequest content in memory */
-
-    if (u->input_filter == NULL) {
-        u->input_filter_init = ngx_http_upstream_non_buffered_filter_init;
-        u->input_filter = ngx_http_upstream_non_buffered_filter;
-        u->input_filter_ctx = r;
-    }
-
-    if (u->input_filter_init(u->input_filter_ctx) == NGX_ERROR) {
-        ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
-        return;
-    }
-
-    n = u->buffer.last - u->buffer.pos;
-
-    if (n) {
-        u->buffer.last = u->buffer.pos;
-
-        u->state->response_length += n;
-
-        if (u->input_filter(u->input_filter_ctx, n) == NGX_ERROR) {
-            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
-            return;
-        }
-    }
-
-    if (u->length == 0) {
-        ngx_http_upstream_finalize_request(r, u, 0);
-        return;
-    }
-
-    u->read_event_handler = ngx_http_upstream_process_body_in_memory;
-
-    ngx_http_upstream_process_body_in_memory(r, u);
+    ngx_http_upstream_send_response(r, u);
 }
 
 
@@ -2776,84 +2736,6 @@ ngx_http_upstream_process_headers(ngx_ht
 
 
 static void
-ngx_http_upstream_process_body_in_memory(ngx_http_request_t *r,
-    ngx_http_upstream_t *u)
-{
-    size_t             size;
-    ssize_t            n;
-    ngx_buf_t         *b;
-    ngx_event_t       *rev;
-    ngx_connection_t  *c;
-
-    c = u->peer.connection;
-    rev = c->read;
-
-    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
-                   "http upstream process body in memory");
-
-    if (rev->timedout) {
-        ngx_connection_error(c, NGX_ETIMEDOUT, "upstream timed out");
-        ngx_http_upstream_finalize_request(r, u, NGX_HTTP_GATEWAY_TIME_OUT);
-        return;
-    }
-
-    b = &u->buffer;
-
-    for ( ;; ) {
-
-        size = b->end - b->last;
-
-        if (size == 0) {
-            ngx_log_error(NGX_LOG_ALERT, c->log, 0,
-                          "upstream buffer is too small to read response");
-            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
-            return;
-        }
-
-        n = c->recv(c, b->last, size);
-
-        if (n == NGX_AGAIN) {
-            break;
-        }
-
-        if (n == 0 || n == NGX_ERROR) {
-            ngx_http_upstream_finalize_request(r, u, n);
-            return;
-        }
-
-        u->state->bytes_received += n;
-        u->state->response_length += n;
-
-        if (u->input_filter(u->input_filter_ctx, n) == NGX_ERROR) {
-            ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
-            return;
-        }
-
-        if (!rev->ready) {
-            break;
-        }
-    }
-
-    if (u->length == 0) {
-        ngx_http_upstream_finalize_request(r, u, 0);
-        return;
-    }
-
-    if (ngx_handle_read_event(rev, 0) != NGX_OK) {
-        ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
-        return;
-    }
-
-    if (rev->active) {
-        ngx_add_timer(rev, u->conf->read_timeout);
-
-    } else if (rev->timer_set) {
-        ngx_del_timer(rev);
-    }
-}
-
-
-static void
 ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u)
 {
     ssize_t                    n;
@@ -4359,12 +4241,6 @@ ngx_http_upstream_finalize_request(ngx_h
 
 #endif
 
-    if (r->subrequest_in_memory
-        && u->headers_in.status_n >= NGX_HTTP_SPECIAL_RESPONSE)
-    {
-        u->buffer.last = u->buffer.pos;
-    }
-
     r->read_event_handler = ngx_http_block_reading;
 
     if (rc == NGX_DECLINED) {