Mercurial > hg > nginx-vendor-current

comparison src/http/ngx_http_file_cache.c @ 632:5b73504dd4ba NGINX_1_1_0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
nginx 1.1.0 *) Feature: cache loader run time decrease. *) Feature: "loader_files", "loader_sleep", and "loader_threshold" options of the "proxy/fastcgi/scgi/uwsgi_cache_path" directives. *) Feature: loading time decrease of configuration with large number of HTTPS sites. *) Feature: now nginx supports ECDHE key exchange ciphers. Thanks to Adrian Kotelba. *) Feature: the "lingering_close" directive. Thanks to Maxim Dounin. *) Bugfix: in closing connection for pipelined requests. Thanks to Maxim Dounin. *) Bugfix: nginx did not disable gzipping if client sent "gzip;q=0" in "Accept-Encoding" request header line. *) Bugfix: in timeout in unbuffered proxied mode. Thanks to Maxim Dounin. *) Bugfix: memory leaks when a "proxy_pass" directive contains variables and proxies to an HTTPS backend. Thanks to Maxim Dounin. *) Bugfix: in parameter validaiton of a "proxy_pass" directive with variables. Thanks to Lanshun Zhou. *) Bugfix: SSL did not work on QNX. Thanks to Maxim Dounin. *) Bugfix: SSL modules could not be built by gcc 4.6 without --with-debug option.
author Igor Sysoev <http://sysoev.ru>
date Mon, 01 Aug 2011 00:00:00 +0400
parents ad6fee8052d7
children 23ef0645ea57
comparison
equal deleted inserted replaced
631:9b978fa3cd33 632:5b73504dd4ba
29 static time_t ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache); 29 static time_t ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache);
30 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache); 30 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache);
31 static void ngx_http_file_cache_delete(ngx_http_file_cache_t *cache, 31 static void ngx_http_file_cache_delete(ngx_http_file_cache_t *cache,
32 ngx_queue_t *q, u_char *name); 32 ngx_queue_t *q, u_char *name);
33 static ngx_int_t 33 static ngx_int_t
34 ngx_http_file_cache_manager_sleep(ngx_http_file_cache_t *cache); 34 ngx_http_file_cache_loader_sleep(ngx_http_file_cache_t *cache);
35 static ngx_int_t ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx, 35 static ngx_int_t ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx,
36 ngx_str_t *path); 36 ngx_str_t *path);
37 static ngx_int_t ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx, 37 static ngx_int_t ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx,
38 ngx_str_t *path); 38 ngx_str_t *path);
39 static ngx_int_t ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, 39 static ngx_int_t ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx,
374 } 374 }
375 375
376 if ((size_t) n < c->header_start) { 376 if ((size_t) n < c->header_start) {
377 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0, 377 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
378 "cache file \"%s\" is too small", c->file.name.data); 378 "cache file \"%s\" is too small", c->file.name.data);
379 return NGX_ERROR; 379 return NGX_DECLINED;
380 } 380 }
381 381
382 h = (ngx_http_file_cache_header_t *) c->buf->pos; 382 h = (ngx_http_file_cache_header_t *) c->buf->pos;
383 383
384 if (h->crc32 != c->crc32) { 384 if (h->crc32 != c->crc32) {
528 rc = NGX_OK; 528 rc = NGX_OK;
529 529
530 goto done; 530 goto done;
531 } 531 }
532 532
533 if (fcn->exists) { 533 if (fcn->exists || fcn->uses >= c->min_uses) {
534 534
535 c->exists = fcn->exists; 535 c->exists = fcn->exists;
536 c->body_start = fcn->body_start; 536 if (fcn->body_start) {
537 c->body_start = fcn->body_start;
538 }
537 539
538 rc = NGX_OK; 540 rc = NGX_OK;
539 541
540 goto done; 542 goto done;
541 } 543 }
542 544
543 if (fcn->uses >= c->min_uses) { 545 rc = NGX_AGAIN;
544
545 c->exists = fcn->exists;
546 c->body_start = fcn->body_start;
547
548 rc = NGX_OK;
549
550 } else {
551 rc = NGX_AGAIN;
552 }
553 546
554 goto done; 547 goto done;
555 } 548 }
556 549
557 fcn = ngx_slab_alloc_locked(cache->shpool, 550 fcn = ngx_slab_alloc_locked(cache->shpool,
856 ngx_http_cache_t *c; 849 ngx_http_cache_t *c;
857 850
858 c = r->cache; 851 c = r->cache;
859 852
860 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 853 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
861 "http file cache send: %s", c->file.name.data); 854 "http file cache send: %s", c->file.name.data);
862 855
863 /* we need to allocate all before the header would be sent */ 856 /* we need to allocate all before the header would be sent */
864 857
865 b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t)); 858 b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t));
866 if (b == NULL) { 859 if (b == NULL) {
1212 1205
1213 if (wait > 0) { 1206 if (wait > 0) {
1214 return wait; 1207 return wait;
1215 } 1208 }
1216 1209
1217 if (ngx_http_file_cache_manager_sleep(cache) != NGX_OK) { 1210 if (ngx_quit || ngx_terminate) {
1218 return next; 1211 return next;
1219 } 1212 }
1220 } 1213 }
1221 } 1214 }
1222 1215
1266 cache->bsize); 1259 cache->bsize);
1267 } 1260 }
1268 1261
1269 1262
1270 static ngx_int_t 1263 static ngx_int_t
1271 ngx_http_file_cache_manager_sleep(ngx_http_file_cache_t *cache) 1264 ngx_http_file_cache_loader_sleep(ngx_http_file_cache_t *cache)
1272 { 1265 {
1273 ngx_msec_t elapsed; 1266 ngx_msec_t elapsed;
1274 1267
1275 if (cache->files++ > 100) { 1268 if (++cache->files >= cache->loader_files) {
1276 1269
1277 ngx_time_update(); 1270 ngx_time_update();
1278 1271
1279 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last)); 1272 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last));
1280 1273
1281 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, 1274 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1282 "http file cache manager time: %M", elapsed); 1275 "http file cache loader time elapsed: %M", elapsed);
1283 1276
1284 if (elapsed > 200) { 1277 if (elapsed >= cache->loader_threshold) {
1285 1278
1286 /* 1279 if (cache->loader_files > 1) {
1287 * if processing 100 files takes more than 200ms, 1280 cache->loader_files /= 2;
1288 * it seems that many operations require disk i/o, 1281 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
1289 * therefore sleep 200ms 1282 "cache %V loader_files decreased to %ui",
1290 */ 1283 &cache->path->name, cache->loader_files);
1291 1284
1292 ngx_msleep(200); 1285 } else {
1293 1286 cache->loader_sleep *= 2;
1294 ngx_time_update(); 1287 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
1295 } 1288 "cache %V loader_sleep increased to %Mms",
1289 &cache->path->name, cache->loader_sleep);
1290 }
1291 }
1292
1293 ngx_msleep(cache->loader_sleep);
1294
1295 ngx_time_update();
1296 1296
1297 cache->last = ngx_current_msec; 1297 cache->last = ngx_current_msec;
1298 cache->files = 0; 1298 cache->files = 0;
1299 } 1299 }
1300 1300
1318 1318
1319 if (ngx_http_file_cache_add_file(ctx, path) != NGX_OK) { 1319 if (ngx_http_file_cache_add_file(ctx, path) != NGX_OK) {
1320 (void) ngx_http_file_cache_delete_file(ctx, path); 1320 (void) ngx_http_file_cache_delete_file(ctx, path);
1321 } 1321 }
1322 1322
1323 return ngx_http_file_cache_manager_sleep(cache); 1323 return ngx_http_file_cache_loader_sleep(cache);
1324 } 1324 }
1325 1325
1326 1326
1327 static ngx_int_t 1327 static ngx_int_t
1328 ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, ngx_str_t *name) 1328 ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, ngx_str_t *name)
1329 { 1329 {
1330 u_char *p; 1330 u_char *p;
1331 ngx_fd_t fd; 1331 ngx_int_t n;
1332 ngx_int_t n; 1332 ngx_uint_t i;
1333 ngx_uint_t i; 1333 ngx_http_cache_t c;
1334 ngx_file_info_t fi; 1334 ngx_http_file_cache_t *cache;
1335 ngx_http_cache_t c;
1336 ngx_http_file_cache_t *cache;
1337 ngx_http_file_cache_header_t h;
1338 1335
1339 if (name->len < 2 * NGX_HTTP_CACHE_KEY_LEN) { 1336 if (name->len < 2 * NGX_HTTP_CACHE_KEY_LEN) {
1340 return NGX_ERROR; 1337 return NGX_ERROR;
1341 } 1338 }
1342 1339
1343 ngx_memzero(&c, sizeof(ngx_http_cache_t)); 1340 if (ctx->size < (off_t) sizeof(ngx_http_file_cache_header_t)) {
1344
1345 fd = ngx_open_file(name->data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
1346
1347 if (fd == NGX_INVALID_FILE) {
1348 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno,
1349 ngx_open_file_n " \"%s\" failed", name->data);
1350 return NGX_ERROR;
1351 }
1352
1353 c.file.fd = fd;
1354 c.file.name = *name;
1355 c.file.log = ctx->log;
1356
1357 n = ngx_read_file(&c.file, (u_char *) &h,
1358 sizeof(ngx_http_file_cache_header_t), 0);
1359 if (n == NGX_ERROR) {
1360 return NGX_ERROR;
1361 }
1362
1363 if ((size_t) n < sizeof(ngx_http_file_cache_header_t)) {
1364 ngx_log_error(NGX_LOG_CRIT, ctx->log, 0, 1341 ngx_log_error(NGX_LOG_CRIT, ctx->log, 0,
1365 "cache file \"%s\" is too small", name->data); 1342 "cache file \"%s\" is too small", name->data);
1366 return NGX_ERROR; 1343 return NGX_ERROR;
1367 } 1344 }
1368 1345
1346 ngx_memzero(&c, sizeof(ngx_http_cache_t));
1369 cache = ctx->data; 1347 cache = ctx->data;
1370 1348
1371 if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) { 1349 c.length = ctx->size;
1372 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno, 1350 c.fs_size = (ctx->fs_size + cache->bsize - 1) / cache->bsize;
1373 ngx_fd_info_n " \"%s\" failed", name->data);
1374
1375 } else {
1376 c.uniq = ngx_file_uniq(&fi);
1377 c.valid_sec = h.valid_sec;
1378 c.valid_msec = h.valid_msec;
1379 c.body_start = h.body_start;
1380 c.length = ngx_file_size(&fi);
1381 c.fs_size = (ngx_file_fs_size(&fi) + cache->bsize - 1) / cache->bsize;
1382 }
1383
1384 if (ngx_close_file(fd) == NGX_FILE_ERROR) {
1385 ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno,
1386 ngx_close_file_n " \"%s\" failed", name->data);
1387 }
1388
1389 if (c.body_start == 0) {
1390 return NGX_ERROR;
1391 }
1392 1351
1393 p = &name->data[name->len - 2 * NGX_HTTP_CACHE_KEY_LEN]; 1352 p = &name->data[name->len - 2 * NGX_HTTP_CACHE_KEY_LEN];
1394 1353
1395 for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) { 1354 for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) {
1396 n = ngx_hextoi(p, 2); 1355 n = ngx_hextoi(p, 2);
1433 1392
1434 ngx_rbtree_insert(&cache->sh->rbtree, &fcn->node); 1393 ngx_rbtree_insert(&cache->sh->rbtree, &fcn->node);
1435 1394
1436 fcn->uses = 1; 1395 fcn->uses = 1;
1437 fcn->count = 0; 1396 fcn->count = 0;
1438 fcn->valid_msec = c->valid_msec; 1397 fcn->valid_msec = 0;
1439 fcn->error = 0; 1398 fcn->error = 0;
1440 fcn->exists = 1; 1399 fcn->exists = 1;
1441 fcn->updating = 0; 1400 fcn->updating = 0;
1442 fcn->deleting = 0; 1401 fcn->deleting = 0;
1443 fcn->uniq = c->uniq; 1402 fcn->uniq = 0;
1444 fcn->valid_sec = c->valid_sec; 1403 fcn->valid_sec = 0;
1445 fcn->body_start = c->body_start; 1404 fcn->body_start = 0;
1446 fcn->fs_size = c->fs_size; 1405 fcn->fs_size = c->fs_size;
1447 1406
1448 cache->sh->size += c->fs_size; 1407 cache->sh->size += c->fs_size;
1449 1408
1450 } else { 1409 } else {
1508 off_t max_size; 1467 off_t max_size;
1509 u_char *last, *p; 1468 u_char *last, *p;
1510 time_t inactive; 1469 time_t inactive;
1511 ssize_t size; 1470 ssize_t size;
1512 ngx_str_t s, name, *value; 1471 ngx_str_t s, name, *value;
1472 ngx_int_t loader_files, loader_sleep, loader_threshold;
1513 ngx_uint_t i, n; 1473 ngx_uint_t i, n;
1514 ngx_http_file_cache_t *cache; 1474 ngx_http_file_cache_t *cache;
1515 1475
1516 cache = ngx_pcalloc(cf->pool, sizeof(ngx_http_file_cache_t)); 1476 cache = ngx_pcalloc(cf->pool, sizeof(ngx_http_file_cache_t));
1517 if (cache == NULL) { 1477 if (cache == NULL) {
1522 if (cache->path == NULL) { 1482 if (cache->path == NULL) {
1523 return NGX_CONF_ERROR; 1483 return NGX_CONF_ERROR;
1524 } 1484 }
1525 1485
1526 inactive = 600; 1486 inactive = 600;
1487 loader_files = 100;
1488 loader_sleep = 50;
1489 loader_threshold = 200;
1527 1490
1528 name.len = 0; 1491 name.len = 0;
1529 size = 0; 1492 size = 0;
1530 max_size = NGX_MAX_OFF_T_VALUE; 1493 max_size = NGX_MAX_OFF_T_VALUE;
1531 1494
1635 } 1598 }
1636 1599
1637 continue; 1600 continue;
1638 } 1601 }
1639 1602
1603 if (ngx_strncmp(value[i].data, "loader_files=", 13) == 0) {
1604
1605 loader_files = ngx_atoi(value[i].data + 13, value[i].len - 13);
1606 if (loader_files == NGX_ERROR) {
1607 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
1608 "invalid loader_files value \"%V\"", &value[i]);
1609 return NGX_CONF_ERROR;
1610 }
1611
1612 continue;
1613 }
1614
1615 if (ngx_strncmp(value[i].data, "loader_sleep=", 13) == 0) {
1616
1617 s.len = value[i].len - 13;
1618 s.data = value[i].data + 13;
1619
1620 loader_sleep = ngx_parse_time(&s, 0);
1621 if (loader_sleep < 0) {
1622 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
1623 "invalid loader_sleep value \"%V\"", &value[i]);
1624 return NGX_CONF_ERROR;
1625 }
1626
1627 continue;
1628 }
1629
1630 if (ngx_strncmp(value[i].data, "loader_threshold=", 17) == 0) {
1631
1632 s.len = value[i].len - 17;
1633 s.data = value[i].data + 17;
1634
1635 loader_threshold = ngx_parse_time(&s, 0);
1636 if (loader_threshold < 0) {
1637 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
1638 "invalid loader_threshold value \"%V\"", &value[i]);
1639 return NGX_CONF_ERROR;
1640 }
1641
1642 continue;
1643 }
1644
1640 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, 1645 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
1641 "invalid parameter \"%V\"", &value[i]); 1646 "invalid parameter \"%V\"", &value[i]);
1642 return NGX_CONF_ERROR; 1647 return NGX_CONF_ERROR;
1643 } 1648 }
1644 1649
1650 } 1655 }
1651 1656
1652 cache->path->manager = ngx_http_file_cache_manager; 1657 cache->path->manager = ngx_http_file_cache_manager;
1653 cache->path->loader = ngx_http_file_cache_loader; 1658 cache->path->loader = ngx_http_file_cache_loader;
1654 cache->path->data = cache; 1659 cache->path->data = cache;
1660 cache->path->conf_file = cf->conf_file->file.name.data;
1661 cache->path->line = cf->conf_file->line;
1662 cache->loader_files = loader_files;
1663 cache->loader_sleep = (ngx_msec_t) loader_sleep;
1664 cache->loader_threshold = (ngx_msec_t) loader_threshold;
1655 1665
1656 if (ngx_add_path(cf, &cache->path) != NGX_OK) { 1666 if (ngx_add_path(cf, &cache->path) != NGX_OK) {
1657 return NGX_CONF_ERROR; 1667 return NGX_CONF_ERROR;
1658 } 1668 }
1659 1669