0
|
1
|
|
2 /*
|
|
3 * Copyright (C) Igor Sysoev
|
|
4 */
|
|
5
|
|
6
|
|
7 #include <ngx_config.h>
|
|
8 #include <ngx_core.h>
|
|
9 #include <ngx_event.h>
|
|
10
|
|
11
|
18
|
12 #if 0
|
|
13 #define NGX_SENDFILE_LIMIT 4096
|
|
14 #endif
|
|
15
|
|
16
|
|
17 #define NGX_NONE 1
|
0
|
18
|
|
19
|
16
|
20 static ngx_inline ngx_int_t
|
406
|
21 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
|
18
|
22 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
|
48
|
23 ngx_chain_t **chain, ngx_chain_t *in);
|
406
|
24 static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
|
|
25 off_t bsize);
|
|
26 static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx,
|
|
27 off_t bsize);
|
|
28 static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx);
|
0
|
29
|
|
30
|
48
|
31 ngx_int_t
|
|
32 ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in)
|
0
|
33 {
|
22
|
34 off_t bsize;
|
146
|
35 ngx_int_t rc, last;
|
0
|
36 ngx_chain_t *cl, *out, **last_out;
|
|
37
|
4
|
38 if (ctx->in == NULL && ctx->busy == NULL) {
|
0
|
39
|
18
|
40 /*
|
|
41 * the short path for the case when the ctx->in and ctx->busy chains
|
|
42 * are empty, the incoming chain is empty too or has the single buf
|
|
43 * that does not require the copy
|
|
44 */
|
0
|
45
|
|
46 if (in == NULL) {
|
|
47 return ctx->output_filter(ctx->filter_ctx, in);
|
|
48 }
|
|
49
|
|
50 if (in->next == NULL
|
18
|
51 #if (NGX_SENDFILE_LIMIT)
|
|
52 && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
|
|
53 #endif
|
406
|
54 && ngx_output_chain_as_is(ctx, in->buf))
|
0
|
55 {
|
|
56 return ctx->output_filter(ctx->filter_ctx, in);
|
|
57 }
|
|
58 }
|
|
59
|
|
60 /* add the incoming buf to the chain ctx->in */
|
|
61
|
|
62 if (in) {
|
18
|
63 if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) {
|
0
|
64 return NGX_ERROR;
|
|
65 }
|
|
66 }
|
|
67
|
|
68 out = NULL;
|
|
69 last_out = &out;
|
48
|
70 last = NGX_NONE;
|
0
|
71
|
|
72 for ( ;; ) {
|
|
73
|
|
74 while (ctx->in) {
|
|
75
|
|
76 /*
|
|
77 * cycle while there are the ctx->in bufs
|
406
|
78 * and there are the free output bufs to copy in
|
0
|
79 */
|
|
80
|
|
81 bsize = ngx_buf_size(ctx->in->buf);
|
|
82
|
|
83 if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) {
|
|
84
|
|
85 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
|
32
|
86 "zero size buf in output "
|
|
87 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
|
|
88 ctx->in->buf->temporary,
|
|
89 ctx->in->buf->recycled,
|
|
90 ctx->in->buf->in_file,
|
|
91 ctx->in->buf->start,
|
|
92 ctx->in->buf->pos,
|
|
93 ctx->in->buf->last,
|
|
94 ctx->in->buf->file,
|
|
95 ctx->in->buf->file_pos,
|
|
96 ctx->in->buf->file_last);
|
0
|
97
|
22
|
98 ngx_debug_point();
|
|
99
|
0
|
100 ctx->in = ctx->in->next;
|
|
101
|
|
102 continue;
|
|
103 }
|
|
104
|
406
|
105 if (ngx_output_chain_as_is(ctx, ctx->in->buf)) {
|
0
|
106
|
|
107 /* move the chain link to the output chain */
|
|
108
|
|
109 cl = ctx->in;
|
|
110 ctx->in = cl->next;
|
|
111
|
|
112 *last_out = cl;
|
|
113 last_out = &cl->next;
|
|
114 cl->next = NULL;
|
|
115
|
|
116 continue;
|
|
117 }
|
|
118
|
|
119 if (ctx->buf == NULL) {
|
|
120
|
406
|
121 rc = ngx_output_chain_align_file_buf(ctx, bsize);
|
0
|
122
|
406
|
123 if (rc == NGX_ERROR) {
|
|
124 return NGX_ERROR;
|
|
125 }
|
0
|
126
|
406
|
127 if (rc != NGX_OK) {
|
0
|
128
|
406
|
129 if (ctx->free) {
|
0
|
130
|
406
|
131 /* get the free buf */
|
0
|
132
|
406
|
133 cl = ctx->free;
|
|
134 ctx->buf = cl->buf;
|
|
135 ctx->free = cl->next;
|
0
|
136
|
406
|
137 ngx_free_chain(ctx->pool, cl);
|
0
|
138
|
406
|
139 } else if (out || ctx->allocated == ctx->bufs.num) {
|
0
|
140
|
406
|
141 break;
|
0
|
142
|
406
|
143 } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
|
0
|
144 return NGX_ERROR;
|
|
145 }
|
|
146 }
|
|
147 }
|
|
148
|
406
|
149 rc = ngx_output_chain_copy_buf(ctx);
|
0
|
150
|
|
151 if (rc == NGX_ERROR) {
|
|
152 return rc;
|
|
153 }
|
|
154
|
|
155 if (rc == NGX_AGAIN) {
|
|
156 if (out) {
|
|
157 break;
|
|
158 }
|
|
159
|
|
160 return rc;
|
|
161 }
|
|
162
|
|
163 /* delete the completed buf from the ctx->in chain */
|
|
164
|
|
165 if (ngx_buf_size(ctx->in->buf) == 0) {
|
|
166 ctx->in = ctx->in->next;
|
|
167 }
|
|
168
|
50
|
169 cl = ngx_alloc_chain_link(ctx->pool);
|
|
170 if (cl == NULL) {
|
10
|
171 return NGX_ERROR;
|
|
172 }
|
50
|
173
|
10
|
174 cl->buf = ctx->buf;
|
|
175 cl->next = NULL;
|
0
|
176 *last_out = cl;
|
|
177 last_out = &cl->next;
|
|
178 ctx->buf = NULL;
|
|
179 }
|
|
180
|
|
181 if (out == NULL && last != NGX_NONE) {
|
58
|
182
|
|
183 if (ctx->in) {
|
|
184 return NGX_AGAIN;
|
|
185 }
|
|
186
|
0
|
187 return last;
|
|
188 }
|
|
189
|
|
190 last = ctx->output_filter(ctx->filter_ctx, out);
|
|
191
|
118
|
192 if (last == NGX_ERROR || last == NGX_DONE) {
|
|
193 return last;
|
|
194 }
|
|
195
|
0
|
196 ngx_chain_update_chains(&ctx->free, &ctx->busy, &out, ctx->tag);
|
|
197 last_out = &out;
|
|
198 }
|
|
199 }
|
|
200
|
|
201
|
16
|
202 static ngx_inline ngx_int_t
|
406
|
203 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
|
0
|
204 {
|
18
|
205 ngx_uint_t sendfile;
|
|
206
|
0
|
207 if (ngx_buf_special(buf)) {
|
406
|
208 return 1;
|
|
209 }
|
|
210
|
|
211 if (buf->in_file && buf->file->directio) {
|
0
|
212 return 0;
|
|
213 }
|
|
214
|
18
|
215 sendfile = ctx->sendfile;
|
|
216
|
|
217 #if (NGX_SENDFILE_LIMIT)
|
|
218
|
|
219 if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) {
|
|
220 sendfile = 0;
|
|
221 }
|
|
222
|
|
223 #endif
|
|
224
|
|
225 if (!sendfile) {
|
4
|
226
|
0
|
227 if (!ngx_buf_in_memory(buf)) {
|
406
|
228 return 0;
|
0
|
229 }
|
|
230
|
|
231 buf->in_file = 0;
|
|
232 }
|
|
233
|
|
234 if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
|
406
|
235 return 0;
|
0
|
236 }
|
|
237
|
|
238 if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
|
406
|
239 return 0;
|
0
|
240 }
|
|
241
|
406
|
242 return 1;
|
0
|
243 }
|
|
244
|
|
245
|
48
|
246 static ngx_int_t
|
|
247 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
|
|
248 ngx_chain_t *in)
|
18
|
249 {
|
|
250 ngx_chain_t *cl, **ll;
|
|
251 #if (NGX_SENDFILE_LIMIT)
|
|
252 ngx_buf_t *b, *buf;
|
|
253 #endif
|
|
254
|
|
255 ll = chain;
|
|
256
|
|
257 for (cl = *chain; cl; cl = cl->next) {
|
|
258 ll = &cl->next;
|
|
259 }
|
|
260
|
|
261 while (in) {
|
|
262
|
50
|
263 cl = ngx_alloc_chain_link(pool);
|
|
264 if (cl == NULL) {
|
18
|
265 return NGX_ERROR;
|
|
266 }
|
|
267
|
|
268 #if (NGX_SENDFILE_LIMIT)
|
|
269
|
|
270 buf = in->buf;
|
|
271
|
|
272 if (buf->in_file
|
|
273 && buf->file_pos < NGX_SENDFILE_LIMIT
|
|
274 && buf->file_last > NGX_SENDFILE_LIMIT)
|
|
275 {
|
406
|
276 /* split a file buf on two bufs by the sendfile limit */
|
|
277
|
50
|
278 b = ngx_calloc_buf(pool);
|
|
279 if (b == NULL) {
|
18
|
280 return NGX_ERROR;
|
|
281 }
|
|
282
|
|
283 ngx_memcpy(b, buf, sizeof(ngx_buf_t));
|
|
284
|
|
285 if (ngx_buf_in_memory(buf)) {
|
|
286 buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos);
|
|
287 b->last = buf->pos;
|
|
288 }
|
|
289
|
|
290 buf->file_pos = NGX_SENDFILE_LIMIT;
|
|
291 b->file_last = NGX_SENDFILE_LIMIT;
|
|
292
|
|
293 cl->buf = b;
|
|
294
|
|
295 } else {
|
|
296 cl->buf = buf;
|
|
297 in = in->next;
|
|
298 }
|
|
299
|
|
300 #else
|
|
301 cl->buf = in->buf;
|
|
302 in = in->next;
|
|
303
|
|
304 #endif
|
|
305
|
|
306 *ll = cl;
|
|
307 ll = &cl->next;
|
|
308 }
|
|
309
|
|
310 *ll = NULL;
|
|
311
|
|
312 return NGX_OK;
|
|
313 }
|
|
314
|
|
315
|
48
|
316 static ngx_int_t
|
406
|
317 ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
|
|
318 {
|
|
319 size_t size;
|
|
320 ngx_buf_t *in;
|
|
321
|
|
322 in = ctx->in->buf;
|
|
323
|
|
324 if (in->file == NULL || !in->file->directio) {
|
|
325 return NGX_DECLINED;
|
|
326 }
|
|
327
|
|
328 ctx->directio = 1;
|
|
329
|
|
330 size = (size_t) (in->file_pos - (in->file_pos & ~511));
|
|
331
|
|
332 if (size == 0) {
|
|
333
|
408
|
334 if (bsize >= (off_t) ctx->bufs.size) {
|
406
|
335 return NGX_DECLINED;
|
|
336 }
|
|
337
|
|
338 size = (size_t) bsize;
|
|
339
|
|
340 } else {
|
|
341 size = 512 - size;
|
|
342
|
408
|
343 if ((off_t) size > bsize) {
|
406
|
344 size = (size_t) bsize;
|
|
345 }
|
|
346 }
|
|
347
|
|
348 ctx->buf = ngx_create_temp_buf(ctx->pool, size);
|
|
349 if (ctx->buf == NULL) {
|
|
350 return NGX_ERROR;
|
|
351 }
|
|
352
|
|
353 /*
|
|
354 * we do not set ctx->buf->tag, because we do not want
|
|
355 * to reuse the buf via ctx->free list
|
|
356 */
|
|
357
|
|
358 return NGX_OK;
|
|
359 }
|
|
360
|
|
361
|
|
362 static ngx_int_t
|
|
363 ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
|
0
|
364 {
|
406
|
365 size_t size;
|
|
366 ngx_buf_t *b, *in;
|
|
367 ngx_uint_t recycled;
|
|
368
|
|
369 in = ctx->in->buf;
|
|
370 size = ctx->bufs.size;
|
|
371 recycled = 1;
|
|
372
|
|
373 if (in->last_in_chain) {
|
|
374
|
|
375 if (bsize < (off_t) size) {
|
|
376
|
|
377 /*
|
|
378 * allocate a small temp buf for a small last buf
|
|
379 * or its small last part
|
|
380 */
|
|
381
|
|
382 size = (size_t) bsize;
|
|
383 recycled = 0;
|
|
384
|
|
385 } else if (!ctx->directio
|
|
386 && ctx->bufs.num == 1
|
|
387 && (bsize < (off_t) (size + size / 4)))
|
|
388 {
|
|
389 /*
|
|
390 * allocate a temp buf that equals to a last buf,
|
|
391 * if there is no directio, the last buf size is lesser
|
|
392 * than 1.25 of bufs.size and the temp buf is single
|
|
393 */
|
|
394
|
|
395 size = (size_t) bsize;
|
|
396 recycled = 0;
|
|
397 }
|
|
398 }
|
|
399
|
|
400 b = ngx_calloc_buf(ctx->pool);
|
|
401 if (b == NULL) {
|
|
402 return NGX_ERROR;
|
|
403 }
|
|
404
|
|
405 if (ctx->directio) {
|
|
406
|
|
407 /*
|
|
408 * allocate block aligned to a disk sector size to enable
|
|
409 * userland buffer direct usage conjunctly with directio
|
|
410 */
|
|
411
|
|
412 b->start = ngx_pmemalign(ctx->pool, size, 512);
|
|
413 if (b->start == NULL) {
|
|
414 return NGX_ERROR;
|
|
415 }
|
|
416
|
|
417 } else {
|
|
418 b->start = ngx_palloc(ctx->pool, size);
|
|
419 if (b->start == NULL) {
|
|
420 return NGX_ERROR;
|
|
421 }
|
|
422 }
|
|
423
|
|
424 b->pos = b->start;
|
|
425 b->last = b->start;
|
|
426 b->end = b->last + size;
|
|
427 b->temporary = 1;
|
|
428 b->tag = ctx->tag;
|
|
429 b->recycled = recycled;
|
|
430
|
|
431 ctx->buf = b;
|
|
432 ctx->allocated++;
|
|
433
|
|
434 return NGX_OK;
|
|
435 }
|
|
436
|
|
437
|
|
438 static ngx_int_t
|
|
439 ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx)
|
|
440 {
|
|
441 off_t size;
|
|
442 ssize_t n;
|
|
443 ngx_buf_t *src, *dst;
|
|
444 ngx_uint_t sendfile;
|
|
445
|
|
446 src = ctx->in->buf;
|
|
447 dst = ctx->buf;
|
0
|
448
|
|
449 size = ngx_buf_size(src);
|
|
450
|
22
|
451 if (size > dst->end - dst->pos) {
|
0
|
452 size = dst->end - dst->pos;
|
|
453 }
|
|
454
|
406
|
455 sendfile = ctx->sendfile & !ctx->directio;
|
|
456
|
18
|
457 #if (NGX_SENDFILE_LIMIT)
|
|
458
|
|
459 if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
|
|
460 sendfile = 0;
|
|
461 }
|
|
462
|
|
463 #endif
|
|
464
|
0
|
465 if (ngx_buf_in_memory(src)) {
|
22
|
466 ngx_memcpy(dst->pos, src->pos, (size_t) size);
|
|
467 src->pos += (size_t) size;
|
|
468 dst->last += (size_t) size;
|
0
|
469
|
4
|
470 if (src->in_file) {
|
|
471
|
|
472 if (sendfile) {
|
|
473 dst->in_file = 1;
|
|
474 dst->file = src->file;
|
|
475 dst->file_pos = src->file_pos;
|
|
476 dst->file_last = src->file_pos + size;
|
|
477
|
|
478 } else {
|
|
479 dst->in_file = 0;
|
|
480 }
|
|
481
|
0
|
482 src->file_pos += size;
|
2
|
483
|
|
484 } else {
|
|
485 dst->in_file = 0;
|
0
|
486 }
|
|
487
|
306
|
488 if (src->pos == src->last) {
|
|
489 dst->flush = src->flush;
|
|
490 dst->last_buf = src->last_buf;
|
0
|
491 }
|
|
492
|
|
493 } else {
|
22
|
494 n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos);
|
0
|
495
|
|
496 if (n == NGX_ERROR) {
|
66
|
497 return (ngx_int_t) n;
|
0
|
498 }
|
|
499
|
|
500 #if (NGX_FILE_AIO_READ)
|
|
501 if (n == NGX_AGAIN) {
|
66
|
502 return (ngx_int_t) n;
|
0
|
503 }
|
|
504 #endif
|
|
505
|
22
|
506 if (n != size) {
|
406
|
507 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
|
398
|
508 ngx_read_file_n " read only %z of %O from file",
|
0
|
509 n, size);
|
|
510 if (n == 0) {
|
|
511 return NGX_ERROR;
|
|
512 }
|
|
513 }
|
|
514
|
|
515 dst->last += n;
|
|
516
|
2
|
517 if (sendfile) {
|
|
518 dst->in_file = 1;
|
|
519 dst->file = src->file;
|
|
520 dst->file_pos = src->file_pos;
|
4
|
521 dst->file_last = src->file_pos + n;
|
2
|
522
|
|
523 } else {
|
0
|
524 dst->in_file = 0;
|
|
525 }
|
|
526
|
4
|
527 src->file_pos += n;
|
|
528
|
308
|
529 if (src->file_pos == src->file_last) {
|
306
|
530 dst->flush = src->flush;
|
|
531 dst->last_buf = src->last_buf;
|
0
|
532 }
|
|
533 }
|
|
534
|
|
535 return NGX_OK;
|
|
536 }
|
|
537
|
|
538
|
48
|
539 ngx_int_t
|
|
540 ngx_chain_writer(void *data, ngx_chain_t *in)
|
0
|
541 {
|
|
542 ngx_chain_writer_ctx_t *ctx = data;
|
|
543
|
372
|
544 off_t size;
|
|
545 ngx_chain_t *cl;
|
|
546 ngx_connection_t *c;
|
|
547
|
|
548 c = ctx->connection;
|
0
|
549
|
22
|
550 for (size = 0; in; in = in->next) {
|
|
551
|
|
552 #if 1
|
|
553 if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) {
|
|
554 ngx_debug_point();
|
|
555 }
|
|
556 #endif
|
|
557
|
|
558 size += ngx_buf_size(in->buf);
|
0
|
559
|
372
|
560 ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0,
|
306
|
561 "chain writer buf fl:%d s:%uO",
|
|
562 in->buf->flush, ngx_buf_size(in->buf));
|
0
|
563
|
50
|
564 cl = ngx_alloc_chain_link(ctx->pool);
|
|
565 if (cl == NULL) {
|
10
|
566 return NGX_ERROR;
|
|
567 }
|
50
|
568
|
10
|
569 cl->buf = in->buf;
|
|
570 cl->next = NULL;
|
0
|
571 *ctx->last = cl;
|
|
572 ctx->last = &cl->next;
|
|
573 }
|
|
574
|
372
|
575 ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
|
10
|
576 "chain writer in: %p", ctx->out);
|
0
|
577
|
22
|
578 for (cl = ctx->out; cl; cl = cl->next) {
|
|
579
|
|
580 #if 1
|
|
581 if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) {
|
|
582 ngx_debug_point();
|
|
583 }
|
|
584
|
|
585 #endif
|
|
586
|
|
587 size += ngx_buf_size(cl->buf);
|
|
588 }
|
|
589
|
372
|
590 if (size == 0 && !c->buffered) {
|
22
|
591 return NGX_OK;
|
|
592 }
|
|
593
|
372
|
594 ctx->out = c->send_chain(c, ctx->out, ctx->limit);
|
0
|
595
|
372
|
596 ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
|
10
|
597 "chain writer out: %p", ctx->out);
|
0
|
598
|
|
599 if (ctx->out == NGX_CHAIN_ERROR) {
|
|
600 return NGX_ERROR;
|
|
601 }
|
|
602
|
|
603 if (ctx->out == NULL) {
|
|
604 ctx->last = &ctx->out;
|
302
|
605
|
372
|
606 if (!c->buffered) {
|
304
|
607 return NGX_OK;
|
|
608 }
|
0
|
609 }
|
|
610
|
|
611 return NGX_AGAIN;
|
|
612 }
|