# HG changeset patch
# User Igor Sysoev <igor@sysoev.ru>
# Date 1067416244 0
# Node ID 84036764e215c4a5a9bb3dc5330e4e36b8d61d97
# Parent  fb61ba77bebae5cbcc3ec3c164e70bc876146965
nginx-0.0.1-2003-10-29-11:30:44 import

diff --git a/src/core/ngx_output_chain.c b/src/core/ngx_output_chain.c
--- a/src/core/ngx_output_chain.c
+++ b/src/core/ngx_output_chain.c
@@ -47,6 +47,7 @@ int ngx_output_chain(ngx_output_chain_ct
     }
 
     last = NGX_NONE;
+    out = NULL;
     last_out = &out;
 
     for ( ;; ) {
@@ -167,7 +168,7 @@ ngx_inline static int ngx_output_chain_n
         return 0;
     }
 
-    if (!ctx->sendfile) {
+    if (!ctx->sendfile && (!(hunk->type & NGX_HUNK_IN_MEMORY))) {
         return 1;
     }
 
diff --git a/src/event/modules/ngx_aio_module.c b/src/event/modules/ngx_aio_module.c
--- a/src/event/modules/ngx_aio_module.c
+++ b/src/event/modules/ngx_aio_module.c
@@ -19,7 +19,7 @@ static int ngx_aio_process_events(ngx_lo
 
 ngx_os_io_t ngx_os_aio = {
     ngx_aio_read,
-    NULL,
+    ngx_aio_read_chain,
     ngx_aio_write,
     ngx_aio_write_chain,
     NGX_HAVE_ZEROCOPY
diff --git a/src/event/modules/ngx_kqueue_module.c b/src/event/modules/ngx_kqueue_module.c
--- a/src/event/modules/ngx_kqueue_module.c
+++ b/src/event/modules/ngx_kqueue_module.c
@@ -437,7 +437,7 @@ static int ngx_kqueue_process_events(ngx
             break;
 
         case EVFILT_AIO:
-            ev->ready = 1;
+            ev->aio_complete = 1;
             ev->active = 0;
 
             ev->event_handler(ev);
diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -60,13 +60,19 @@ struct ngx_event_s {
     unsigned         instance:1;
 
     /*
-     * event was passed or would be passed to a kernel;
-     * the posted aio operation.
+     * the event was passed or would be passed to a kernel;
+     * aio mode: 1 - the posted aio operation,
+     *           0 - the complete aio operation or no aio operation.
      */
     unsigned         active:1;
 
-    /* ready event; the complete aio operation */
+    /*
+     * the ready event;
+     * in aio mode "ready" is always set - it makes things simple
+     * to learn whether the aio operation complete use aio_complete flag
+     */
     unsigned         ready:1;
+    unsigned         aio_complete:1;
 
     unsigned         eof:1;
     unsigned         error:1;
@@ -338,15 +344,8 @@ int ngx_event_post_acceptex(ngx_listenin
 
 
 
-ngx_inline static int ngx_handle_read_event(ngx_event_t *rev)
+ngx_inline static int ngx_handle_read_event(ngx_event_t *rev, int close)
 {
-    if (ngx_event_flags & (NGX_USE_AIO_EVENT|NGX_USE_EDGE_EVENT)) {
-
-        /* aio, iocp, epoll */
-
-        return NGX_OK;
-    }
-
     if (ngx_event_flags & NGX_USE_CLEAR_EVENT) {
 
         /* kqueue */
@@ -359,26 +358,31 @@ ngx_inline static int ngx_handle_read_ev
         }
 
         return NGX_OK;
-    }
 
-    /* select, poll, /dev/poll */
+    } else if (ngx_event_flags & NGX_USE_LEVEL_EVENT) {
+
+        /* select, poll, /dev/poll */
 
-    if (!rev->active && !rev->ready) {
-        if (ngx_add_event(rev, NGX_READ_EVENT, NGX_LEVEL_EVENT) == NGX_ERROR) {
-            return NGX_ERROR;
+        if (!rev->active && !rev->ready) {
+            if (ngx_add_event(rev, NGX_READ_EVENT, NGX_LEVEL_EVENT)
+                                                                == NGX_ERROR) {
+                return NGX_ERROR;
+            }
+
+            return NGX_OK;
         }
 
-        return NGX_OK;
+        if (rev->active && (rev->ready || close)) {
+            if (ngx_del_event(rev, NGX_READ_EVENT, close ? NGX_CLOSE_EVENT : 0)
+                                                                == NGX_ERROR) {
+                return NGX_ERROR;
+            }
+
+            return NGX_OK;
+        }
     }
 
-    if (rev->active && (rev->ready || rev->eof)) {
-        if (ngx_del_event(rev, NGX_READ_EVENT, rev->eof ? NGX_CLOSE_EVENT : 0)
-                                                                == NGX_ERROR) {
-            return NGX_ERROR;
-        }
-
-        return NGX_OK;
-    }
+    /* aio, iocp, epoll, rt signals */
 
     return NGX_OK;
 }
@@ -411,13 +415,6 @@ ngx_inline static int ngx_handle_level_r
 
 ngx_inline static int ngx_handle_write_event(ngx_event_t *wev, int lowat)
 {
-    if (ngx_event_flags & (NGX_USE_AIO_EVENT|NGX_USE_EDGE_EVENT)) {
-
-        /* aio, iocp, epoll */
-
-        return NGX_OK;
-    }
-
     if (ngx_event_flags & NGX_USE_CLEAR_EVENT) {
 
         /* kqueue */
@@ -437,25 +434,30 @@ ngx_inline static int ngx_handle_write_e
         }
 
         return NGX_OK;
-    }
 
-    /* select, poll, /dev/poll */
+    } else if (ngx_event_flags & NGX_USE_LEVEL_EVENT) {
+
+        /* select, poll, /dev/poll */
 
-    if (!wev->active && !wev->ready) {
-        if (ngx_add_event(wev, NGX_WRITE_EVENT, NGX_LEVEL_EVENT) == NGX_ERROR) {
-            return NGX_ERROR;
+        if (!wev->active && !wev->ready) {
+            if (ngx_add_event(wev, NGX_WRITE_EVENT, NGX_LEVEL_EVENT)
+                                                                == NGX_ERROR) {
+                return NGX_ERROR;
+            }
+
+            return NGX_OK;
         }
 
-        return NGX_OK;
+        if (wev->active && wev->ready) {
+            if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) == NGX_ERROR) {
+                return NGX_ERROR;
+            }
+
+            return NGX_OK;
+        }
     }
 
-    if (wev->active && wev->ready) {
-        if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) == NGX_ERROR) {
-            return NGX_ERROR;
-        }
-
-        return NGX_OK;
-    }
+    /* aio, iocp, epoll, rt signals */
 
     return NGX_OK;
 }
diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c
--- a/src/event/ngx_event_accept.c
+++ b/src/event/ngx_event_accept.c
@@ -172,9 +172,11 @@ ngx_log_debug(ev->log, "ADDR %s" _ ls->l
         c->fd = s;
         c->unexpected_eof = 1;
         wev->write = 1;
+        wev->ready = 1;
 
-        if ((ngx_event_flags & NGX_USE_AIO_EVENT) == 0) {
-            wev->ready = 1;
+        if (ngx_event_flags & (NGX_USE_AIO_EVENT|NGX_USE_EDGE_EVENT)) {
+            /* aio, iocp, epoll */
+            rev->ready = 1;
         }
 
         c->ctx = ls->ctx;
diff --git a/src/event/ngx_event_connect.c b/src/event/ngx_event_connect.c
--- a/src/event/ngx_event_connect.c
+++ b/src/event/ngx_event_connect.c
@@ -202,7 +202,19 @@ ngx_log_debug(pc->log, "CONNECT: %s" _ p
         }
     }
 
-    /* TODO: epoll, aio, iocp */
+    if (ngx_event_flags & NGX_USE_AIO_EVENT) {
+        /* aio, iocp */
+        rev->ready = 1;
+ 
+#if 1
+        /* TODO: NGX_EINPROGRESS */
+
+        wev->ready = 1;
+        return NGX_OK;
+#endif
+    }
+
+    /* TODO: epoll */
 
     if (ngx_event_flags & NGX_USE_CLEAR_EVENT) {     /* kqueue */
         event = NGX_CLEAR_EVENT;
diff --git a/src/event/ngx_event_pipe.c b/src/event/ngx_event_pipe.c
--- a/src/event/ngx_event_pipe.c
+++ b/src/event/ngx_event_pipe.c
@@ -19,6 +19,8 @@ static int ngx_event_pipe_drain_chains(n
 
 int ngx_event_pipe(ngx_event_pipe_t *p, int do_write)
 {
+    ngx_event_t  *rev, *wev;
+
     for ( ;; ) {
         if (do_write) {
             if (ngx_event_pipe_write_to_downstream(p) == NGX_ABORT) {
@@ -40,15 +42,26 @@ int ngx_event_pipe(ngx_event_pipe_t *p, 
         do_write = 1;
     }
 
-    if (ngx_handle_read_event(p->upstream->read) == NGX_ERROR) {
+    rev = p->upstream->read;
+
+    if (ngx_handle_read_event(rev, (rev->eof || rev->error)) == NGX_ERROR) {
         return NGX_ABORT;
     }
 
-    if (ngx_handle_write_event(p->downstream->write,
-                                           /* TODO: lowat */ 0) == NGX_ERROR) {
+    if (rev->active) {
+        ngx_add_timer(rev, p->read_timeout);
+    }
+
+    wev = p->downstream->write;
+
+    if (ngx_handle_write_event(wev, p->send_lowat) == NGX_ERROR) {
         return NGX_ABORT;
     }
 
+    if (wev->active) {
+        ngx_add_timer(wev, p->send_timeout);
+    }
+
     return NGX_OK;
 }
 
@@ -112,7 +125,6 @@ int ngx_event_pipe_read_upstream(ngx_eve
 
                             ngx_log_error(NGX_LOG_ERR, p->log,
                                           p->upstream->read->kq_errno,
-                                          /* TODO: ngx_readv_chain_n */
                                           "readv() failed");
                         }
 
@@ -213,8 +225,6 @@ int ngx_event_pipe_read_upstream(ngx_eve
                 break;
             }
 
-            ngx_log_debug(p->log, "HUNK_FREE: %d" _ chain->hunk->num);
-
             n = ngx_recv_chain(p->upstream, chain);
 
             ngx_log_debug(p->log, "recv_chain: %d" _ n);
@@ -343,8 +353,6 @@ int ngx_event_pipe_write_to_downstream(n
                 ngx_event_pipe_free_shadow_raw_hunk(&p->free_raw_hunks,
                                                     cl->hunk);
 
-ngx_log_debug(p->log, "HUNK OUT: %d %x" _ cl->hunk->num _ cl->hunk->type);
-
             } else if (!p->cachable && p->in) {
                 cl = p->in;
 
@@ -356,8 +364,6 @@ ngx_log_debug(p->log, "HUNK OUT: %d %x" 
 
                 p->in = p->in->next;
 
-ngx_log_debug(p->log, "HUNK IN: %d" _ cl->hunk->num);
-
             } else {
                 break;
             }
diff --git a/src/event/ngx_event_pipe.h b/src/event/ngx_event_pipe.h
--- a/src/event/ngx_event_pipe.h
+++ b/src/event/ngx_event_pipe.h
@@ -59,6 +59,10 @@ struct ngx_event_pipe_s {
     ngx_connection_t  *upstream;
     ngx_connection_t  *downstream;
 
+    ngx_msec_t         read_timeout;
+    ngx_msec_t         send_timeout;
+    ssize_t            send_lowat;
+
     ngx_pool_t        *pool;
     ngx_log_t         *log;
 
diff --git a/src/http/modules/ngx_http_gzip_filter.c b/src/http/modules/ngx_http_gzip_filter.c
--- a/src/http/modules/ngx_http_gzip_filter.c
+++ b/src/http/modules/ngx_http_gzip_filter.c
@@ -190,18 +190,23 @@ static int ngx_http_gzip_header_filter(n
                         sizeof(ngx_http_gzip_ctx_t), NGX_ERROR);
     ctx->request = r;
 
-    ngx_test_null(r->headers_out.content_encoding,
-                  ngx_push_table(r->headers_out.headers),
-                  NGX_ERROR);
+    if (!(r->headers_out.content_encoding =
+                   ngx_http_add_header(&r->headers_out, ngx_http_headers_out)))
+    {
+        return NGX_ERROR;
+    }
 
     r->headers_out.content_encoding->key.len = 0;
     r->headers_out.content_encoding->key.data = NULL;
-    r->headers_out.content_encoding->value.len = 4;
+    r->headers_out.content_encoding->value.len = sizeof("gzip") - 1;
     r->headers_out.content_encoding->value.data = "gzip";
 
     ctx->length = r->headers_out.content_length_n;
     r->headers_out.content_length_n = -1;
-    r->headers_out.content_length = NULL;
+    if (r->headers_out.content_length) {
+        r->headers_out.content_length->key.len = 0;
+        r->headers_out.content_length = NULL;
+    }
     r->filter |= NGX_HTTP_FILTER_NEED_IN_MEMORY;
 
     return ngx_http_next_header_filter(r);
diff --git a/src/http/modules/ngx_http_range_filter.c b/src/http/modules/ngx_http_range_filter.c
--- a/src/http/modules/ngx_http_range_filter.c
+++ b/src/http/modules/ngx_http_range_filter.c
@@ -96,9 +96,12 @@ static int ngx_http_range_header_filter(
         || r->headers_in.range->value.len < 7
         || ngx_strncasecmp(r->headers_in.range->value.data, "bytes=", 6) != 0)
     {
-        ngx_test_null(r->headers_out.accept_ranges,
-                      ngx_push_table(r->headers_out.headers),
-                      NGX_ERROR);
+
+        if (!(r->headers_out.accept_ranges =
+                   ngx_http_add_header(&r->headers_out, ngx_http_headers_out)))
+        {
+            return NGX_ERROR;
+        }
 
         r->headers_out.accept_ranges->key.len = sizeof("Accept-Ranges") - 1;
         r->headers_out.accept_ranges->key.data = "Accept-Ranges";
diff --git a/src/http/modules/proxy/ngx_http_proxy_handler.c b/src/http/modules/proxy/ngx_http_proxy_handler.c
--- a/src/http/modules/proxy/ngx_http_proxy_handler.c
+++ b/src/http/modules/proxy/ngx_http_proxy_handler.c
@@ -475,6 +475,11 @@ static void ngx_http_proxy_send_request(
 
                     /* rc == NGX_OK */
 
+                    if (c->read->ready) {
+                        /* post aio operation */
+                        ngx_http_proxy_process_upstream_status_line(c->read);
+                    }
+
                     if (ngx_handle_level_write_event(c->write) == NGX_ERROR) {
                         ngx_http_proxy_finalize_request(p,
                                                NGX_HTTP_INTERNAL_SERVER_ERROR);
@@ -742,8 +747,8 @@ static void ngx_http_proxy_process_upstr
                 }
             }
 
-            ngx_log_debug(c->log, "HTTP proxy header: '%s: %s'" _
-                          h->key.data _ h->value.data);
+            ngx_log_debug(c->log, "HTTP proxy header: %08X '%s: %s'" _
+                          h _ h->key.data _ h->value.data);
 
             continue;
 
@@ -791,10 +796,6 @@ static ssize_t ngx_http_proxy_read_upstr
     n = p->header_in->last - p->header_in->pos;
 
     if (n > 0) {
-#if 0
-        /* TODO THINK */
-        rev->ready = 0;
-#endif
         return n;
     }
 
@@ -804,7 +805,7 @@ static ssize_t ngx_http_proxy_read_upstr
     if (n == NGX_AGAIN) {
         ngx_add_timer(rev, p->lcf->read_timeout);
 
-        if (ngx_handle_read_event(rev) == NGX_ERROR) {
+        if (ngx_handle_read_event(rev, 0) == NGX_ERROR) {
             ngx_http_proxy_finalize_request(p, NGX_HTTP_INTERNAL_SERVER_ERROR);
             return NGX_ERROR;
         }
@@ -829,10 +830,11 @@ static ssize_t ngx_http_proxy_read_upstr
 
 static void ngx_http_proxy_send_response(ngx_http_proxy_ctx_t *p)
 {
-    int                  rc, i;
-    ngx_table_elt_t     *ch, *ph;
-    ngx_event_pipe_t    *ep;
-    ngx_http_request_t  *r;
+    int                        rc, i;
+    ngx_table_elt_t           *ch, *h;
+    ngx_event_pipe_t          *ep;
+    ngx_http_request_t        *r;
+    ngx_http_core_loc_conf_t  *clcf;
 
     r = p->request;
 
@@ -843,26 +845,26 @@ static void ngx_http_proxy_send_response
 
     /* copy an upstream header to r->headers_out */
 
-    ph = (ngx_table_elt_t *) p->headers_in.headers->elts;
+    h = p->headers_in.headers->elts;
     for (i = 0; i < p->headers_in.headers->nelts; i++) {
 
-        if (&ph[i] == p->headers_in.connection) {
+        if (&h[i] == p->headers_in.connection) {
             continue;
         }
 
         if (p->accel) {
-            if (&ph[i] == p->headers_in.date
-                || &ph[i] == p->headers_in.accept_ranges) {
+            if (&h[i] == p->headers_in.date
+                || &h[i] == p->headers_in.accept_ranges) {
                 continue;
             }
 
-            if (&ph[i] == p->headers_in.server && !p->lcf->pass_server) {
+            if (&h[i] == p->headers_in.server && !p->lcf->pass_server) {
                 continue;
             }
         }
 
-        if (&ph[i] == p->headers_in.content_type) {
-            r->headers_out.content_type = &ph[i];
+        if (&h[i] == p->headers_in.content_type) {
+            r->headers_out.content_type = &h[i];
             r->headers_out.content_type->key.len = 0;
             continue;
         }
@@ -873,7 +875,7 @@ static void ngx_http_proxy_send_response
             return;
         }
 
-        *ch = ph[i];
+        *ch = h[i];
 
         /*
          * ngx_http_header_filter() output the following headers
@@ -883,17 +885,17 @@ static void ngx_http_proxy_send_response
          *     r->headers_out.content_length
          */
 
-        if (&ph[i] == p->headers_in.server) {
+        if (&h[i] == p->headers_in.server) {
             r->headers_out.server = ch;
             continue;
         }
 
-        if (&ph[i] == p->headers_in.date) {
+        if (&h[i] == p->headers_in.date) {
             r->headers_out.date = ch;
             continue;
         }
 
-        if (&ph[i] == p->headers_in.content_length) {
+        if (&h[i] == p->headers_in.content_length) {
 
             r->headers_out.content_length_n =
                              ngx_atoi(p->headers_in.content_length->value.data,
@@ -977,6 +979,12 @@ static void ngx_http_proxy_send_response
         r->sendfile = 1;
     }
 
+    clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+    ep->read_timeout = p->lcf->read_timeout;
+    ep->send_timeout = clcf->send_timeout;
+    ep->send_lowat = clcf->send_lowat;
+
     p->event_pipe = ep;
 
     p->upstream.connection->read->event_handler = ngx_http_proxy_process_body;
diff --git a/src/http/modules/proxy/ngx_http_proxy_handler.h b/src/http/modules/proxy/ngx_http_proxy_handler.h
--- a/src/http/modules/proxy/ngx_http_proxy_handler.h
+++ b/src/http/modules/proxy/ngx_http_proxy_handler.h
@@ -44,6 +44,8 @@ typedef struct {
 
 
 typedef struct {
+    ngx_table_t      *headers;   /* it must be first field */
+
     ngx_table_elt_t  *date;
     ngx_table_elt_t  *server;
     ngx_table_elt_t  *connection;
@@ -53,8 +55,6 @@ typedef struct {
     ngx_table_elt_t  *accept_ranges;
 
     off_t             content_length_n;
-
-    ngx_table_t      *headers;
 } ngx_http_proxy_headers_in_t;
 
 
diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h
--- a/src/http/ngx_http.h
+++ b/src/http/ngx_http.h
@@ -60,6 +60,9 @@ int ngx_http_special_response_handler(ng
 
 time_t ngx_http_parse_time(char *value, size_t len);
 size_t ngx_http_get_time(char *buf, time_t t);
+ngx_table_elt_t *ngx_http_add_header(void *header,
+                                     ngx_http_header_t *http_headers);
+
 
 
 int ngx_http_discard_body(ngx_http_request_t *r);
diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c
--- a/src/http/ngx_http_core_module.c
+++ b/src/http/ngx_http_core_module.c
@@ -4,7 +4,9 @@
 #include <ngx_event.h>
 #include <ngx_http.h>
 #include <nginx.h>
-
+#if __FreeBSD__
+#include <ngx_freebsd_init.h>
+#endif
 
 
 static void ngx_http_phase_event_handler(ngx_event_t *rev);
@@ -31,6 +33,10 @@ static char *ngx_set_server_name(ngx_con
                                  void *conf);
 static char *ngx_set_error_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
 
+static char *ngx_http_lowat_check(ngx_conf_t *cf, void *post, void *data);
+
+static ngx_conf_post_t  ngx_http_lowat_post = { ngx_http_lowat_check } ;
+
 
 static ngx_command_t  ngx_http_core_commands[] = {
 
@@ -151,6 +157,13 @@ static ngx_command_t  ngx_http_core_comm
      offsetof(ngx_http_core_loc_conf_t, send_timeout),
      NULL},
 
+    {ngx_string("send_lowat"),
+     NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+     ngx_conf_set_size_slot,
+     NGX_HTTP_LOC_CONF_OFFSET,
+     offsetof(ngx_http_core_loc_conf_t, send_lowat),
+     &ngx_http_lowat_post},
+
     {ngx_string("keepalive_timeout"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
      ngx_conf_set_msec_slot,
@@ -506,6 +519,49 @@ int ngx_http_delay_handler(ngx_http_requ
 #endif
 
 
+ngx_table_elt_t *ngx_http_add_header(void *header,
+                                     ngx_http_header_t *http_headers)
+{
+    int               i, j;
+    char             *prev;
+    ngx_table_t      *headers;
+    ngx_table_elt_t  *h, *new;
+
+    headers = *(ngx_table_t **) header;
+
+    prev = headers->elts;
+
+    if (!(new = ngx_push_table(headers))) {
+        return NULL;
+    }
+
+    if (prev == headers->elts) {
+        return new;
+    }
+
+    h = headers->elts;
+    for (i = 0; i < headers->nelts; i++) {
+        if (h[i].key.len == 0) {
+            continue;
+        }
+
+        for (j = 0; http_headers[j].name.len != 0; j++) {
+            if (http_headers[j].name.len != h[i].key.len) {
+                continue;
+            }
+
+            if (ngx_strcasecmp(http_headers[j].name.data, h[i].key.data) == 0) {
+                *((ngx_table_elt_t **)
+                      ((char *) &header + http_headers[j].offset)) = &h[i];
+                break;
+            }
+        }
+    }
+
+    return new;
+}
+
+
 static int ngx_http_core_init(ngx_cycle_t *cycle)
 {
 #if 0
@@ -852,6 +908,7 @@ static void *ngx_http_core_create_loc_co
     lcf->client_body_timeout = NGX_CONF_UNSET;
     lcf->sendfile = NGX_CONF_UNSET;
     lcf->send_timeout = NGX_CONF_UNSET;
+    lcf->send_lowat = NGX_CONF_UNSET;
     lcf->discarded_buffer_size = NGX_CONF_UNSET;
     lcf->keepalive_timeout = NGX_CONF_UNSET;
     lcf->lingering_time = NGX_CONF_UNSET;
@@ -925,6 +982,7 @@ static char *ngx_http_core_merge_loc_con
                               prev->client_body_timeout, 10000);
     ngx_conf_merge_value(conf->sendfile, prev->sendfile, 0);
     ngx_conf_merge_msec_value(conf->send_timeout, prev->send_timeout, 10000);
+    ngx_conf_merge_size_value(conf->send_lowat, prev->send_lowat, 0);
     ngx_conf_merge_size_value(conf->discarded_buffer_size,
                               prev->discarded_buffer_size, 1500);
     ngx_conf_merge_msec_value(conf->keepalive_timeout,
@@ -1063,3 +1121,29 @@ static char *ngx_set_error_log(ngx_conf_
 
     return NGX_CONF_OK;
 }
+
+
+static char *ngx_http_lowat_check(ngx_conf_t *cf, void *post, void *data)
+{
+    int *np = data;
+
+#if (HAVE_LOWAT_EVENT)
+
+    if (*np >= ngx_freebsd_net_inet_tcp_sendspace) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                           "\"send_lowat\" must be less than %d "
+                           "(sysctl net.inet.tcp.sendspace)",
+                           ngx_freebsd_net_inet_tcp_sendspace);
+
+        return NGX_CONF_ERROR;
+    }
+
+#else
+
+    ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+                       "\"send_lowat\" is not supported, ignored");
+
+#endif
+
+    return NGX_CONF_OK;
+}
diff --git a/src/http/ngx_http_header_filter.c b/src/http/ngx_http_header_filter.c
--- a/src/http/ngx_http_header_filter.c
+++ b/src/http/ngx_http_header_filter.c
@@ -102,9 +102,8 @@ static int ngx_http_header_filter(ngx_ht
         r->header_only = 1;
     }
 
-    /* 9 is for "HTTP/1.x ", 2 is for trailing "\r\n"
-       and 2 is for end of header */
-    len = 9 + 2 + 2;
+    /* 2 is for trailing "\r\n" and 2 is for "\r\n" in the end of header */
+    len = sizeof("HTTP/1.x ") - 1 + 2 + 2;
 
     /* status line */
     if (r->headers_out.status_line.len) {
@@ -150,29 +149,29 @@ static int ngx_http_header_filter(ngx_ht
         len += r->headers_out.date->key.len
                + r->headers_out.date->value.len + 2;
     } else {
-        /* "Date: ... \r\n" */
-        len += 37;
+        len += sizeof("Date: Mon, 28 Sep 1970 00:00:00 GMT" CRLF) - 1;
     }
 
     if (r->headers_out.content_range && r->headers_out.content_range->value.len)
     {
-        len += 15 + r->headers_out.content_range->value.len + 2;
+        len += sizeof("Content-Range: ") - 1
+               + r->headers_out.content_range->value.len + 2;
     }
 
     if (r->headers_out.content_length == NULL) {
         if (r->headers_out.content_length_n >= 0) {
-            /* "Content-Length: ... \r\n", 2^64 is 20 characters */
-            len += 48;
+                                            /* 2^64 */
+            len += sizeof("Content-Length: 18446744073709551616" CRLF) - 1;
         }
     }
 
     if (r->headers_out.content_type && r->headers_out.content_type->value.len) {
         r->headers_out.content_type->key.len = 0;
-        len += 14 + r->headers_out.content_type->value.len + 2;
+        len += sizeof("Content-Type: ") - 1
+               + r->headers_out.content_type->value.len + 2;
 
         if (r->headers_out.charset.len) {
-            /* "; charset= ... " */
-            len += 10 + r->headers_out.charset.len;
+            len += sizeof("; charset=") - 1 + r->headers_out.charset.len;
         }
     }
 
@@ -187,9 +186,8 @@ static int ngx_http_header_filter(ngx_ht
         && r->headers_out.location->value.data[0] == '/')
     {
         r->headers_out.location->key.len = 0;
-        /* "Location: http:// ... \r\n" */
-        len += 17 + r->server_name->len
-               + r->headers_out.location->value.len + 2;
+        len += sizeof("Location: http://") - 1,
+               + r->server_name->len + r->headers_out.location->value.len + 2;
 
         if (r->port != 80) {
             len += r->port_name->len;
@@ -201,24 +199,20 @@ static int ngx_http_header_filter(ngx_ht
                + r->headers_out.last_modified->value.len + 2;
 
     } else if (r->headers_out.last_modified_time != -1) {
-        /* "Last-Modified: ... \r\n" */
-        len += 46;
+        len += sizeof("Last-Modified: Mon, 28 Sep 1970 00:00:00 GMT" CRLF) - 1;
     }
 
     if (r->chunked) {
-        /* "Transfer-Encoding: chunked\r\n" */
-        len += 28;
+        len += sizeof("Transfer-Encoding: chunked" CRLF) - 1;
     }
 
     if (r->keepalive) {
-        /* "Connection: keep-alive\r\n" */
-        len += 24;
+        len += sizeof("Connection: keep-alive" CRLF) - 1;
     } else {
-        /* "Connection: close\r\n" */
-        len += 19;
+        len += sizeof("Connection: closed" CRLF) - 1;
     }
 
-    header = (ngx_table_elt_t *) r->headers_out.headers->elts;
+    header = r->headers_out.headers->elts;
     for (i = 0; i < r->headers_out.headers->nelts; i++) {
         if (header[i].key.len == 0) {
             continue;
@@ -230,7 +224,7 @@ static int ngx_http_header_filter(ngx_ht
     ngx_test_null(h, ngx_create_temp_hunk(r->pool, len, 0, 64), NGX_ERROR);
 
     /* "HTTP/1.x " */
-    h->last = ngx_cpymem(h->last, "HTTP/1.1 ", 9);
+    h->last = ngx_cpymem(h->last, "HTTP/1.1 ", sizeof("HTTP/1.x ") - 1);
 
     /* status line */
     if (r->headers_out.status_line.len) {
@@ -248,7 +242,7 @@ static int ngx_http_header_filter(ngx_ht
     }
 
     if (!(r->headers_out.date && r->headers_out.date->key.len)) {
-        h->last = ngx_cpymem(h->last, "Date: ", 6);
+        h->last = ngx_cpymem(h->last, "Date: ", sizeof("Date: ") - 1);
         h->last += ngx_http_get_time(h->last, time(NULL));
         *(h->last++) = CR; *(h->last++) = LF;
     }
@@ -256,7 +250,8 @@ static int ngx_http_header_filter(ngx_ht
 
     if (r->headers_out.content_range && r->headers_out.content_range->value.len)
     {
-        h->last = ngx_cpymem(h->last, "Content-Range: ", 15);
+        h->last = ngx_cpymem(h->last, "Content-Range: ",
+                             sizeof("Content-Range: ") - 1);
         h->last = ngx_cpymem(h->last, r->headers_out.content_range->value.data,
                              r->headers_out.content_range->value.len);
         *(h->last++) = CR; *(h->last++) = LF;
@@ -265,19 +260,22 @@ static int ngx_http_header_filter(ngx_ht
     if (r->headers_out.content_length == NULL) {
         /* 2^64 is 20 characters  */
         if (r->headers_out.content_length_n >= 0) {
-            h->last += ngx_snprintf(h->last, 49,
-                                    "Content-Length: " OFF_FMT CRLF,
-                                    r->headers_out.content_length_n);
+            h->last += ngx_snprintf(h->last,
+                            sizeof("Content-Length: 18446744073709551616" CRLF),
+                            "Content-Length: " OFF_FMT CRLF,
+                            r->headers_out.content_length_n);
         }
     }
 
     if (r->headers_out.content_type && r->headers_out.content_type->value.len) {
-        h->last = ngx_cpymem(h->last, "Content-Type: ", 14);
+        h->last = ngx_cpymem(h->last, "Content-Type: ",
+                             sizeof("Content-Type: ") - 1);
         h->last = ngx_cpymem(h->last, r->headers_out.content_type->value.data,
                              r->headers_out.content_type->value.len);
 
         if (r->headers_out.charset.len) {
-            h->last = ngx_cpymem(h->last, "; charset=", 10);
+            h->last = ngx_cpymem(h->last, "; charset=",
+                                 sizeof("; charset=") - 1);
             h->last = ngx_cpymem(h->last, r->headers_out.charset.data,
                                  r->headers_out.charset.len);
         }
@@ -288,7 +286,8 @@ static int ngx_http_header_filter(ngx_ht
     if (r->headers_out.content_encoding
         && r->headers_out.content_encoding->value.len)
     {
-        h->last = ngx_cpymem(h->last, "Content-Encoding: ", 18);
+        h->last = ngx_cpymem(h->last, "Content-Encoding: ",
+                             sizeof("Content-Encoding: ") - 1);
         h->last = ngx_cpymem(h->last,
                              r->headers_out.content_encoding->value.data,
                              r->headers_out.content_encoding->value.len);
@@ -300,7 +299,8 @@ static int ngx_http_header_filter(ngx_ht
         && r->headers_out.location->value.len
         && r->headers_out.location->value.data[0] == '/')
     {
-        h->last = ngx_cpymem(h->last, "Location: http://", 17);
+        h->last = ngx_cpymem(h->last, "Location: http://",
+                             sizeof("Location: http://") - 1);
         h->last = ngx_cpymem(h->last, r->server_name->data,
                              r->server_name->len);
         if (r->port != 80) {
@@ -317,21 +317,25 @@ static int ngx_http_header_filter(ngx_ht
     if (!(r->headers_out.last_modified && r->headers_out.last_modified->key.len)
         && r->headers_out.last_modified_time != -1)
     {
-        h->last = ngx_cpymem(h->last, "Last-Modified: ", 15);
+        h->last = ngx_cpymem(h->last, "Last-Modified: ",
+                             sizeof("Last-Modified: ") - 1);
         h->last += ngx_http_get_time(h->last,
-                                         r->headers_out.last_modified_time);
+                                            r->headers_out.last_modified_time);
         *(h->last++) = CR; *(h->last++) = LF;
     }
 
     if (r->chunked) {
-        h->last = ngx_cpymem(h->last, "Transfer-Encoding: chunked" CRLF, 28);
+        h->last = ngx_cpymem(h->last, "Transfer-Encoding: chunked" CRLF,
+                             sizeof("Transfer-Encoding: chunked" CRLF) - 1);
     }
 
     if (r->keepalive) {
-        h->last = ngx_cpymem(h->last, "Connection: keep-alive" CRLF, 24);
+        h->last = ngx_cpymem(h->last, "Connection: keep-alive" CRLF,
+                             sizeof("Connection: keep-alive" CRLF) - 1);
 
     } else {
-        h->last = ngx_cpymem(h->last, "Connection: close" CRLF, 19);
+        h->last = ngx_cpymem(h->last, "Connection: close" CRLF,
+                             sizeof("Connection: close" CRLF) - 1);
     }
 
     for (i = 0; i < r->headers_out.headers->nelts; i++) {
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -44,7 +44,7 @@ static char *client_header_errors[] = {
 
 
 
-static ngx_http_header_t headers_in[] = {
+ngx_http_header_t ngx_http_headers_in[] = {
     { ngx_string("Host"), offsetof(ngx_http_headers_in_t, host) },
     { ngx_string("Connection"), offsetof(ngx_http_headers_in_t, connection) },
     { ngx_string("If-Modified-Since"),
@@ -66,6 +66,27 @@ static ngx_http_header_t headers_in[] = 
 };
 
 
+ngx_http_header_t ngx_http_headers_out[] = {
+    { ngx_string("Server"), offsetof(ngx_http_headers_out_t, server) },
+    { ngx_string("Date"), offsetof(ngx_http_headers_out_t, date) },
+    { ngx_string("Content-Type"),
+                             offsetof(ngx_http_headers_out_t, content_type) },
+    { ngx_string("Content-Length"),
+                           offsetof(ngx_http_headers_out_t, content_length) },
+    { ngx_string("Content-Encoding"),
+                         offsetof(ngx_http_headers_out_t, content_encoding) },
+
+    /* Location */
+
+    { ngx_string("Last-Modified"),
+                            offsetof(ngx_http_headers_out_t, last_modified) },
+    { ngx_string("Accept-Ranges"),
+                            offsetof(ngx_http_headers_out_t, accept_ranges) },
+
+    { ngx_null_string, 0 }
+};
+
+
 static void ngx_http_dummy(ngx_event_t *wev)
 {
     return;
@@ -106,20 +127,14 @@ void ngx_http_init_connection(ngx_connec
     rev->event_handler = ngx_http_init_request;
 
     if (rev->ready) {
-        /* deferred accept */
+        /* deferred accept, aio, iocp, epoll */
         ngx_http_init_request(rev);
         return;
     }
 
     ngx_add_timer(rev, c->listening->post_accept_timeout);
 
-    if (ngx_event_flags & (NGX_USE_AIO_EVENT|NGX_USE_EDGE_EVENT)) {
-        /* aio, iocp, epoll */
-        ngx_http_init_request(rev);
-        return;
-    }
-
-    if (ngx_handle_read_event(rev) == NGX_ERROR) {
+    if (ngx_handle_read_event(rev, 0) == NGX_ERROR) {
         ngx_http_close_connection(c);
         return;
     }
@@ -256,7 +271,7 @@ ngx_log_debug(rev->log, "IN: %08x" _ in_
         return;
     }
 
-    r->headers_out.headers = ngx_create_table(r->pool, 10);
+    r->headers_out.headers = ngx_create_table(r->pool, 1);
     if (r->headers_out.headers == NULL) {
         ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
         ngx_http_close_connection(c);
@@ -451,7 +466,7 @@ static void ngx_http_process_request_lin
         lctx = c->log->data;
         lctx->action = "reading client request headers";
         lctx->url = r->unparsed_uri.data;
-        r->headers_in.headers = ngx_create_table(r->pool, 10);
+        r->headers_in.headers = ngx_create_table(r->pool, 1);
 
         if (cscf->large_client_header
             && r->header_in->pos == r->header_in->last)
@@ -560,8 +575,8 @@ static void ngx_http_process_request_hea
 
             /* a header line has been parsed successfully */
 
-            h = ngx_push_table(r->headers_in.headers);
-            if (h == NULL) {
+            if (!(h = ngx_http_add_header(&r->headers_in, ngx_http_headers_in)))
+            {
                 ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
                 ngx_http_close_connection(c);
                 return;
@@ -593,14 +608,16 @@ static void ngx_http_process_request_hea
                 h->value.data[h->value.len] = '\0';
             }
 
-            for (i = 0; headers_in[i].name.len != 0; i++) {
-                if (headers_in[i].name.len != h->key.len) {
+            for (i = 0; ngx_http_headers_in[i].name.len != 0; i++) {
+                if (ngx_http_headers_in[i].name.len != h->key.len) {
                     continue;
                 }
 
-                if (ngx_strcasecmp(headers_in[i].name.data, h->key.data) == 0) {
-                    *((ngx_table_elt_t **)
-                        ((char *) &r->headers_in + headers_in[i].offset)) = h;
+                if (ngx_strcasecmp(ngx_http_headers_in[i].name.data,
+                                   h->key.data) == 0)
+                {
+                    *((ngx_table_elt_t **) ((char *) &r->headers_in
+                                         + ngx_http_headers_in[i].offset)) = h;
                     break;
                 }
             }
@@ -692,10 +709,6 @@ static ssize_t ngx_http_read_request_hea
     n = r->header_in->last - r->header_in->pos;
 
     if (n > 0) {
-#if 0
-        /* TODO: THINK - AIO ??? */
-        rev->ready = 0;
-#endif
         return n;
     }
 
@@ -709,7 +722,7 @@ static ssize_t ngx_http_read_request_hea
             r->header_timeout_set = 1;
         }
 
-        if (ngx_handle_read_event(rev) == NGX_ERROR) {
+        if (ngx_handle_read_event(rev, 0) == NGX_ERROR) {
             ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
             ngx_http_close_connection(r->connection);
             return NGX_ERROR;
@@ -1157,7 +1170,7 @@ static void ngx_http_set_keepalive(ngx_h
         c->tcp_nopush = 0;
     }
 
-    if (rev->ready || (ngx_event_flags & NGX_USE_AIO_EVENT)) {
+    if (rev->ready) {
         ngx_http_keepalive_handler(rev);
     }
 }
@@ -1256,7 +1269,7 @@ static void ngx_http_set_lingering_close
         return;
     }
 
-    if (rev->ready || (ngx_event_flags & NGX_USE_AIO_EVENT)) {
+    if (rev->ready) {
         ngx_http_lingering_close_handler(rev);
     }
 }
diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h
--- a/src/http/ngx_http_request.h
+++ b/src/http/ngx_http_request.h
@@ -68,6 +68,8 @@ typedef struct {
 
 
 typedef struct {
+    ngx_table_t      *headers;   /* it must be first field */
+
     ngx_table_elt_t  *host;
     ngx_table_elt_t  *connection;
     ngx_table_elt_t  *if_modified_since;
@@ -83,8 +85,6 @@ typedef struct {
     ssize_t           content_length_n;
     size_t            connection_type;
     ssize_t           keep_alive_n;
-
-    ngx_table_t      *headers;
 } ngx_http_headers_in_t;
 
 
@@ -108,6 +108,8 @@ typedef struct {
 
 
 typedef struct {
+    ngx_table_t      *headers;   /* it must be first field */
+
     int               status;
     ngx_str_t         status_line;
 
@@ -124,8 +126,6 @@ typedef struct {
     ngx_str_t         charset;
     ngx_array_t       ranges;
 
-    ngx_table_t      *headers;
-
     off_t             content_length_n;
     char             *etag;
     time_t            date_time;
@@ -231,4 +231,9 @@ struct ngx_http_request_s {
 };
 
 
+extern ngx_http_header_t ngx_http_headers_in[];
+extern ngx_http_header_t ngx_http_headers_out[];
+
+
+
 #endif /* _NGX_HTTP_REQUEST_H_INCLUDED_ */
diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -112,7 +112,7 @@ static void ngx_http_read_client_request
         clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
         ngx_add_timer(rev, clcf->client_body_timeout);
 
-        if (ngx_handle_read_event(rev) == NGX_ERROR) {
+        if (ngx_handle_read_event(rev, 0) == NGX_ERROR) {
             ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
         }
 
diff --git a/src/http/ngx_http_special_response.c b/src/http/ngx_http_special_response.c
--- a/src/http/ngx_http_special_response.c
+++ b/src/http/ngx_http_special_response.c
@@ -218,9 +218,11 @@ int ngx_http_special_response_handler(ng
                                           + sizeof(error_tail) - 1
                                           + sizeof(msie_stub) - 1;
 
-        ngx_test_null(r->headers_out.content_type,
-                      ngx_push_table(r->headers_out.headers),
-                      NGX_HTTP_INTERNAL_SERVER_ERROR);
+        if (!(r->headers_out.content_type =
+                   ngx_http_add_header(&r->headers_out, ngx_http_headers_out)))
+        {
+            return NGX_ERROR;
+        }
 
         r->headers_out.content_type->key.len = 12;
         r->headers_out.content_type->key.data = "Content-Type";
@@ -245,7 +247,9 @@ int ngx_http_special_response_handler(ng
     out = NULL;
     ll = NULL;
 
-    ngx_test_null(h, ngx_calloc_hunk(r->pool), NGX_ERROR);
+    if (!(h = ngx_calloc_hunk(r->pool))) {
+        return NGX_ERROR;
+    }
     h->type = NGX_HUNK_MEMORY|NGX_HUNK_IN_MEMORY;
     h->pos = error_pages[err].data;
     h->last = error_pages[err].data + error_pages[err].len;
@@ -254,7 +258,9 @@ int ngx_http_special_response_handler(ng
     ngx_chain_add_link(out, ll, cl);
 
 
-    ngx_test_null(h, ngx_calloc_hunk(r->pool), NGX_ERROR);
+    if (!(h = ngx_calloc_hunk(r->pool))) {
+        return NGX_ERROR;
+    }
     h->type = NGX_HUNK_MEMORY|NGX_HUNK_IN_MEMORY;
     h->pos = error_tail;
     h->last = error_tail + sizeof(error_tail) - 1;
@@ -270,7 +276,9 @@ int ngx_http_special_response_handler(ng
         && error != NGX_HTTP_REQUEST_URI_TOO_LARGE
        )
     {
-        ngx_test_null(h, ngx_calloc_hunk(r->pool), NGX_ERROR);
+        if (!(h = ngx_calloc_hunk(r->pool))) {
+            return NGX_ERROR;
+        }
         h->type = NGX_HUNK_MEMORY|NGX_HUNK_IN_MEMORY;
         h->pos = msie_stub;
         h->last = msie_stub + sizeof(msie_stub) - 1;
diff --git a/src/os/unix/ngx_aio.h b/src/os/unix/ngx_aio.h
--- a/src/os/unix/ngx_aio.h
+++ b/src/os/unix/ngx_aio.h
@@ -6,6 +6,7 @@
 
 
 ssize_t ngx_aio_read(ngx_connection_t *c, char *buf, size_t size);
+ssize_t ngx_aio_read_chain(ngx_connection_t *c, ngx_chain_t *cl);
 ssize_t ngx_aio_write(ngx_connection_t *c, char *buf, size_t size);
 ngx_chain_t *ngx_aio_write_chain(ngx_connection_t *c, ngx_chain_t *in);
 
diff --git a/src/os/unix/ngx_aio_read.c b/src/os/unix/ngx_aio_read.c
--- a/src/os/unix/ngx_aio_read.c
+++ b/src/os/unix/ngx_aio_read.c
@@ -10,102 +10,89 @@
 
 
 /*
-    The data is ready - 3 syscalls:
-        aio_read(), aio_error(), aio_return()
-    The data is not ready - 4 (kqueue) or 5 syscalls:
-        aio_read(), aio_error(), notifiction,
-                                             aio_error(), aio_return()
-                                             aio_cancel(), aio_error()
-*/
-
+ * the ready data requires 3 syscalls:
+ *     aio_write(), aio_error(), aio_return()
+ * the non-ready data requires 4 (kqueue) or 5 syscalls:
+ *     aio_write(), aio_error(), notifiction, aio_error(), aio_return()
+ *                               timeout, aio_cancel(), aio_error()
+ */
 
 ssize_t ngx_aio_read(ngx_connection_t *c, char *buf, size_t size)
 {
-    int           rc, first, canceled;
-    ngx_event_t  *ev;
+    int           n;
+    ngx_event_t  *rev;
+
+    rev = c->read;
 
-    ev = c->read;
+    if (rev->active) {
+        ngx_log_error(NGX_LOG_ALERT, rev->log, 0, "SECOND AIO POST");
+        return NGX_AGAIN;
+    }
 
-    canceled = 0;
+    if (!rev->aio_complete) {
+        ngx_memzero(&rev->aiocb, sizeof(struct aiocb));
 
-    if (ev->timedout) {
-        ngx_set_socket_errno(NGX_ETIMEDOUT);
-        ngx_log_error(NGX_LOG_ERR, ev->log, 0, "aio_read() timed out");
+        rev->aiocb.aio_fildes = c->fd;
+        rev->aiocb.aio_buf = buf;
+        rev->aiocb.aio_nbytes = size;
 
-        rc = aio_cancel(c->fd, &ev->aiocb);
-        if (rc == -1) {
-            ngx_log_error(NGX_LOG_CRIT, ev->log, ngx_errno,
-                          "aio_cancel() failed");
+#if (HAVE_KQUEUE)
+        rev->aiocb.aio_sigevent.sigev_notify_kqueue = ngx_kqueue;
+        rev->aiocb.aio_sigevent.sigev_notify = SIGEV_KEVENT;
+        rev->aiocb.aio_sigevent.sigev_value.sigval_ptr = rev;
+#endif
+
+        if (aio_read(&rev->aiocb) == -1) {
+            ngx_log_error(NGX_LOG_CRIT, rev->log, ngx_errno,
+                          "aio_read() failed");
+            rev->error = 1;
             return NGX_ERROR;
         }
 
-        ngx_log_debug(ev->log, "aio_cancel: %d" _ rc);
+        ngx_log_debug(rev->log, "aio_read: OK");
 
-        canceled = 1;
-
-        ev->ready = 1;
+        rev->active = 1;
     }
 
-    first = 0;
-
-    if (!ev->ready) {
-        ngx_memzero(&ev->aiocb, sizeof(struct aiocb));
-
-        ev->aiocb.aio_fildes = c->fd;
-        ev->aiocb.aio_buf = buf;
-        ev->aiocb.aio_nbytes = size;
-
-#if (HAVE_KQUEUE)
-        ev->aiocb.aio_sigevent.sigev_notify_kqueue = ngx_kqueue;
-        ev->aiocb.aio_sigevent.sigev_notify = SIGEV_KEVENT;
-        ev->aiocb.aio_sigevent.sigev_value.sigval_ptr = ev;
-#endif
+    rev->aio_complete = 0;
 
-        if (aio_read(&ev->aiocb) == -1) {
-            ngx_log_error(NGX_LOG_CRIT, ev->log, ngx_errno,
-                          "aio_read() failed");
-            return NGX_ERROR;
-        }
-
-        ngx_log_debug(ev->log, "aio_read: OK");
-
-        ev->active = 1;
-        first = 1;
-    }
-
-    ev->ready = 0;
-
-    rc = aio_error(&ev->aiocb);
-    if (rc == -1) {
-        ngx_log_error(NGX_LOG_CRIT, ev->log, ngx_errno, "aio_error() failed");
+    n = aio_error(&rev->aiocb);
+    if (n == -1) {
+        ngx_log_error(NGX_LOG_ALERT, rev->log, ngx_errno, "aio_error() failed");
+        rev->error = 1;
         return NGX_ERROR;
     }
 
-    if (rc != 0) {
-        if (rc == NGX_EINPROGRESS) {
-            if (!first) {
-                ngx_log_error(NGX_LOG_CRIT, ev->log, rc,
+    if (n != 0) {
+        if (n == NGX_EINPROGRESS) {
+            if (!rev->active) {
+                ngx_log_error(NGX_LOG_ALERT, rev->log, n,
                               "aio_read() still in progress");
             }
             return NGX_AGAIN;
         }
 
-        if (rc == NGX_ECANCELED && canceled) {
-            return NGX_ERROR;
-        }
-
-        ngx_log_error(NGX_LOG_CRIT, ev->log, rc, "aio_read() failed");
+        ngx_log_error(NGX_LOG_CRIT, rev->log, n, "aio_read() failed");
+        rev->error = 1;
         return NGX_ERROR;
     }
 
-    rc = aio_return(&ev->aiocb);
-    if (rc == -1) {
-        ngx_log_error(NGX_LOG_CRIT, ev->log, ngx_errno, "aio_return() failed");
+    n = aio_return(&rev->aiocb);
+    if (n == -1) {
+        ngx_log_error(NGX_LOG_ALERT, rev->log, ngx_errno,
+                      "aio_return() failed");
 
+        rev->error = 1;
         return NGX_ERROR;
     }
 
-    ngx_log_debug(ev->log, "aio_read: %d" _ rc);
+    rev->active = 0;
+
+    ngx_log_debug(rev->log, "aio_read: %d" _ n);
 
-    return rc;
+    if (n == 0) {
+        rev->eof = 1;
+    }
+
+    return n;
 }
diff --git a/src/os/unix/ngx_aio_read_chain.c b/src/os/unix/ngx_aio_read_chain.c
new file mode 100644
--- /dev/null
+++ b/src/os/unix/ngx_aio_read_chain.c
@@ -0,0 +1,57 @@
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_aio.h>
+
+
+ssize_t ngx_aio_read_chain(ngx_connection_t *c, ngx_chain_t *cl)
+{
+    int           n;
+    char         *buf, *prev;
+    size_t        size, total;
+    ngx_err_t     err;
+
+    total = 0;
+
+    while (cl) {
+
+        /* we can post the single aio operation only */
+
+        if (c->read->active) {
+            return total ? total : NGX_AGAIN;
+        }
+
+        buf = cl->hunk->pos;
+        prev = buf;
+        size = 0;
+
+        /* coalesce the neighbouring hunks */
+
+        while (cl && prev == cl->hunk->pos) {
+            size += cl->hunk->last - cl->hunk->pos;
+            prev = cl->hunk->last;
+            cl = cl->next;
+        }
+
+        n = ngx_aio_read(c, buf, size);
+
+        ngx_log_debug(c->log, "aio_read: %d" _ n);
+
+        if (n == NGX_AGAIN) {
+            return total ? total : NGX_AGAIN;
+        }
+
+        if (n == NGX_ERROR) {
+            return NGX_ERROR;
+        }
+
+        if (n > 0) {
+            total += n;
+        }
+
+        ngx_log_debug(c->log, "aio_read total: %d" _ total);
+    }
+
+    return total ? total : NGX_AGAIN;
+}
diff --git a/src/os/unix/ngx_aio_write.c b/src/os/unix/ngx_aio_write.c
--- a/src/os/unix/ngx_aio_write.c
+++ b/src/os/unix/ngx_aio_write.c
@@ -10,107 +10,89 @@
 
 
 /*
-    The data is ready - 3 syscalls:
-        aio_write(), aio_error(), aio_return()
-    The data is not ready - 4 (kqueue) or 5 syscalls:
-        aio_write(), aio_error(), notifiction,
-                                             aio_error(), aio_return()
-                                             aio_cancel(), aio_error()
-*/
+ * the ready data requires 3 syscalls:
+ *     aio_write(), aio_error(), aio_return()
+ * the non-ready data requires 4 (kqueue) or 5 syscalls:
+ *     aio_write(), aio_error(), notifiction, aio_error(), aio_return()
+ *                               timeout, aio_cancel(), aio_error()
+ */
 
 ssize_t ngx_aio_write(ngx_connection_t *c, char *buf, size_t size)
 {
-    int           rc, first, canceled;
-    ngx_event_t  *ev;
+    int           n;
+    ngx_event_t  *wev;
 
-    ev = c->write;
+    wev = c->write;
 
-    canceled = 0;
+    if (wev->active) {
+        return NGX_AGAIN;
+    }
 
-ngx_log_debug(ev->log, "aio: ev->ready: %d" _ ev->ready);
-ngx_log_debug(ev->log, "aio: aiocb: %08x" _ &ev->aiocb);
+ngx_log_debug(wev->log, "aio: wev->aio_complete: %d" _ wev->aio_complete);
 
-#if 0
-    if (ev->timedout) {
-        ngx_set_socket_errno(NGX_ETIMEDOUT);
-        ngx_log_error(NGX_LOG_ERR, ev->log, 0, "aio_write() timed out");
+    if (!wev->aio_complete) {
+        ngx_memzero(&wev->aiocb, sizeof(struct aiocb));
 
-        rc = aio_cancel(c->fd, &ev->aiocb);
-        if (rc == -1) {
-            ngx_log_error(NGX_LOG_CRIT, ev->log, ngx_errno,
-                          "aio_cancel() failed");
-            return NGX_ERROR;
-        }
+        wev->aiocb.aio_fildes = c->fd;
+        wev->aiocb.aio_buf = buf;
+        wev->aiocb.aio_nbytes = size;
 
-        ngx_log_debug(ev->log, "aio_cancel: %d" _ rc);
-
-        canceled = 1;
-
-        ev->ready = 1;
-    }
+#if (HAVE_KQUEUE)
+        wev->aiocb.aio_sigevent.sigev_notify_kqueue = ngx_kqueue;
+        wev->aiocb.aio_sigevent.sigev_notify = SIGEV_KEVENT;
+        wev->aiocb.aio_sigevent.sigev_value.sigval_ptr = wev;
 #endif
 
-    first = 0;
-
-    if (!ev->ready) {
-        ngx_memzero(&ev->aiocb, sizeof(struct aiocb));
-
-        ev->aiocb.aio_fildes = c->fd;
-        ev->aiocb.aio_buf = buf;
-        ev->aiocb.aio_nbytes = size;
-
-#if (HAVE_KQUEUE)
-        ev->aiocb.aio_sigevent.sigev_notify_kqueue = ngx_kqueue;
-        ev->aiocb.aio_sigevent.sigev_notify = SIGEV_KEVENT;
-        ev->aiocb.aio_sigevent.sigev_value.sigval_ptr = ev;
-#endif
-
-        if (aio_write(&ev->aiocb) == -1) {
-            ngx_log_error(NGX_LOG_CRIT, ev->log, ngx_errno,
+        if (aio_write(&wev->aiocb) == -1) {
+            ngx_log_error(NGX_LOG_CRIT, wev->log, ngx_errno,
                           "aio_write() failed");
             return NGX_ERROR;
         }
 
-        ngx_log_debug(ev->log, "aio_write: OK");
+        ngx_log_debug(wev->log, "aio_write: OK");
 
-        ev->active = 1;
-        first = 1;
+        wev->active = 1;
     }
 
-    ev->ready = 0;
+    wev->aio_complete = 0;
 
-    rc = aio_error(&ev->aiocb);
-    if (rc == -1) {
-        ngx_log_error(NGX_LOG_CRIT, ev->log, ngx_errno, "aio_error() failed");
+    n = aio_error(&wev->aiocb);
+    if (n == -1) {
+        ngx_log_error(NGX_LOG_CRIT, wev->log, ngx_errno, "aio_error() failed");
+        wev->error = 1;
         return NGX_ERROR;
     }
 
-    if (rc != 0) {
-        if (rc == NGX_EINPROGRESS) {
-            if (!first) {
-                ngx_log_error(NGX_LOG_CRIT, ev->log, rc,
+    if (n != 0) {
+        if (n == NGX_EINPROGRESS) {
+            if (!wev->active) {
+                ngx_log_error(NGX_LOG_ALERT, wev->log, n,
                               "aio_write() still in progress");
             }
             return NGX_AGAIN;
         }
 
-        if (rc == NGX_ECANCELED && canceled) {
-            return NGX_ERROR;
-        }
-
-        ngx_log_error(NGX_LOG_CRIT, ev->log, rc, "aio_write() failed");
+        ngx_log_error(NGX_LOG_CRIT, wev->log, n, "aio_write() failed");
+        wev->error = 1;
         return NGX_ERROR;
     }
 
-    rc = aio_return(&ev->aiocb);
-    if (rc == -1) {
-        ngx_log_error(NGX_LOG_CRIT, ev->log, ngx_errno, "aio_return() failed");
+    n = aio_return(&wev->aiocb);
+    if (n == -1) {
+        ngx_log_error(NGX_LOG_ALERT, wev->log, ngx_errno,
+                      "aio_return() failed");
 
+        wev->error = 1;
         return NGX_ERROR;
     }
 
-    ev->active = 0;
-    ngx_log_debug(ev->log, "aio_write: %d" _ rc);
+    wev->active = 0;
+
+    ngx_log_debug(wev->log, "aio_write: %d" _ n);
 
-    return rc;
+    if (n == 0) {
+        wev->eof = 1;
+    }
+
+    return n;
 }
diff --git a/src/os/unix/ngx_aio_write_chain.c b/src/os/unix/ngx_aio_write_chain.c
--- a/src/os/unix/ngx_aio_write_chain.c
+++ b/src/os/unix/ngx_aio_write_chain.c
@@ -7,68 +7,74 @@
 
 ngx_chain_t *ngx_aio_write_chain(ngx_connection_t *c, ngx_chain_t *in)
 {
-    int              rc;
-    char            *buf, *prev;
-    off_t            sent;
-    size_t           size;
-    ngx_err_t        err;
-    ngx_chain_t     *ce;
+    int           n;
+    char         *buf, *prev;
+    off_t         sent;
+    size_t        size;
+    ngx_err_t     err;
+    ngx_chain_t  *cl;
 
     sent = 0;
-    ce = in;
+    cl = in;
+
+    while (cl) {
 
-    while (ce) {
+        if (cl->hunk->last - cl->hunk->pos == 0) {
+            cl = cl->next;
+            continue;
+        }
 
         /* we can post the single aio operation only */
 
         if (c->write->active) {
-            return ce;
+            return cl;
         }
 
-        buf = prev = ce->hunk->pos;
+        buf = cl->hunk->pos;
+        prev = buf;
         size = 0;
 
-        /* coalesce the neighbouring chain entries */
+        /* coalesce the neighbouring hunks */
 
-        while (ce && prev == ce->hunk->pos) {
-            size += ce->hunk->last - ce->hunk->pos;
-            prev = ce->hunk->last;
-            ce = ce->next;
+        while (cl && prev == cl->hunk->pos) {
+            size += cl->hunk->last - cl->hunk->pos;
+            prev = cl->hunk->last;
+            cl = cl->next;
         }
 
-        rc = ngx_aio_write(c, buf, size);
+        n = ngx_aio_write(c, buf, size);
 
 #if (NGX_DEBUG_WRITE_CHAIN)
-        ngx_log_debug(c->log, "aio_write rc: %d" _ rc);
+        ngx_log_debug(c->log, "aio_write: %d" _ n);
 #endif
 
-        if (rc == NGX_ERROR) {
+        if (n == NGX_ERROR) {
             return NGX_CHAIN_ERROR;
         }
 
-        if (rc > 0) {
-            sent += rc;
-            c->sent += rc;
+        if (n > 0) {
+            sent += n;
+            c->sent += n;
         }
 
 #if (NGX_DEBUG_WRITE_CHAIN)
         ngx_log_debug(c->log, "aio_write sent: " OFF_FMT _ c->sent);
 #endif
 
-        for (ce = in; ce; ce = ce->next) {
+        for (cl = in; cl; cl = cl->next) {
 
-            if (sent >= ce->hunk->last - ce->hunk->pos) {
-                sent -= ce->hunk->last - ce->hunk->pos;
-                ce->hunk->pos = ce->hunk->last;
+            if (sent >= cl->hunk->last - cl->hunk->pos) {
+                sent -= cl->hunk->last - cl->hunk->pos;
+                cl->hunk->pos = cl->hunk->last;
 
                 continue;
             }
 
-            ce->hunk->pos += sent;
+            cl->hunk->pos += sent;
 
             break;
         }
     }
 
-    return ce;
+    return cl;
 }
diff --git a/src/os/unix/ngx_readv_chain.c b/src/os/unix/ngx_readv_chain.c
--- a/src/os/unix/ngx_readv_chain.c
+++ b/src/os/unix/ngx_readv_chain.c
@@ -7,13 +7,14 @@
 ssize_t ngx_readv_chain(ngx_connection_t *c, ngx_chain_t *chain)
 {
     char          *prev;
-    ssize_t        n;
+    ssize_t        n, size;
     struct iovec  *iov;
     ngx_err_t      err;
     ngx_array_t    io;
 
     prev = NULL;
     iov = NULL;
+    size = 0;
 
     ngx_init_array(io, c->pool, 10, sizeof(struct iovec), NGX_ERROR);
 
@@ -29,6 +30,7 @@ ssize_t ngx_readv_chain(ngx_connection_t
             iov->iov_len = chain->hunk->end - chain->hunk->last;
         }
 
+        size += chain->hunk->end - chain->hunk->last;
         prev = chain->hunk->end;
         chain = chain->next;
     }
@@ -42,7 +44,6 @@ ngx_log_debug(c->log, "recv: %d:%d" _ io
 
     } else if (n == -1) {
         c->read->ready = 0;
-        c->read->error = 1;
 
         err = ngx_errno;
         if (err == NGX_EAGAIN) {
@@ -50,8 +51,12 @@ ngx_log_debug(c->log, "recv: %d:%d" _ io
             return NGX_AGAIN;
         }
 
+        c->read->error = 1;
         ngx_log_error(NGX_LOG_ERR, c->log, err, "readv() failed");
         return NGX_ERROR;
+
+    } else if (n < size) {
+        c->read->ready = 0;
     }
 
     return n;