# HG changeset patch # User Igor Sysoev # Date 1256578337 0 # Node ID 8c76116820f36fae04dfeee898288048e31754b2 # Parent 1f3cd08ebb82a974c2052501b4b2fae8675c98e8 merge r3024, r3025, r3028, r3033, r3034, r3035, r3036: allow cross device temporary files atomic copying diff --git a/src/core/ngx_file.c b/src/core/ngx_file.c --- a/src/core/ngx_file.c +++ b/src/core/ngx_file.c @@ -8,8 +8,9 @@ #include -static ngx_atomic_uint_t ngx_temp_number; -static ngx_atomic_uint_t ngx_random_number; +static ngx_atomic_t temp_number = 0; +ngx_atomic_t *ngx_temp_number = &temp_number; +ngx_atomic_int_t ngx_random_number = 123456; ssize_t @@ -99,13 +100,7 @@ ngx_create_temp_file(ngx_file_t *file, n continue; } - if ((path->level[0] == 0) - || (err != NGX_ENOENT -#if (NGX_WIN32) - && err != NGX_ENOTDIR -#endif - )) - { + if ((path->level[0] == 0) || (err != NGX_ENOPATH)) { ngx_log_error(NGX_LOG_CRIT, file->log, err, ngx_open_tempfile_n " \"%s\" failed", file->name.data); @@ -211,22 +206,16 @@ ngx_create_full_path(u_char *dir, ngx_ui } -void -ngx_init_temp_number(void) -{ - ngx_temp_number = 0; - ngx_random_number = 123456; -} - - ngx_atomic_uint_t ngx_next_temp_number(ngx_uint_t collision) { - if (collision) { - ngx_temp_number += ngx_random_number; - } + ngx_atomic_uint_t n, add; + + add = collision ? ngx_random_number : 1; - return ngx_temp_number++; + n = ngx_atomic_fetch_add(ngx_temp_number, add); + + return n + add; } @@ -530,7 +519,9 @@ ngx_create_pathes(ngx_cycle_t *cycle, ng ngx_int_t ngx_ext_rename_file(ngx_str_t *src, ngx_str_t *to, ngx_ext_rename_file_t *ext) { - ngx_err_t err; + u_char *name; + ngx_err_t err; + ngx_copy_file_t cf; #if !(NGX_WIN32) @@ -560,14 +551,8 @@ ngx_ext_rename_file(ngx_str_t *src, ngx_ err = ngx_errno; - if (err -#if (NGX_WIN32) - == ERROR_PATH_NOT_FOUND -#else - == NGX_ENOENT -#endif - ) - { + if (err == NGX_ENOPATH) { + if (!ext->create_path) { goto failed; } @@ -586,7 +571,6 @@ ngx_ext_rename_file(ngx_str_t *src, ngx_ } err = ngx_errno; - goto failed; } #if (NGX_WIN32) @@ -607,6 +591,53 @@ ngx_ext_rename_file(ngx_str_t *src, ngx_ #endif + if (err == NGX_EXDEV) { + + cf.size = -1; + cf.buf_size = 0; + cf.access = ext->access; + cf.time = ext->time; + cf.log = ext->log; + + name = ngx_alloc(to->len + 1 + 10 + 1, ext->log); + if (name == NULL) { + return NGX_ERROR; + } + + (void) ngx_sprintf(name, "%*s.%010uD%Z", to->len, to->data, + (uint32_t) ngx_next_temp_number(0)); + + if (ngx_copy_file(src->data, name, &cf) == NGX_OK) { + + if (ngx_rename_file(name, to->data) != NGX_FILE_ERROR) { + ngx_free(name); + + if (ngx_delete_file(src->data) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_CRIT, ext->log, ngx_errno, + ngx_delete_file_n " \"%s\" failed", + src->data); + return NGX_ERROR; + } + + return NGX_OK; + } + + ngx_log_error(NGX_LOG_CRIT, ext->log, ngx_errno, + ngx_rename_file_n " \"%s\" to \"%s\" failed", + name, to->data); + + if (ngx_delete_file(name) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_CRIT, ext->log, ngx_errno, + ngx_delete_file_n " \"%s\" failed", name); + + } + } + + ngx_free(name); + + err = 0; + } + failed: if (ext->delete_file) { @@ -616,15 +647,141 @@ failed: } } - if (err && ext->log_rename_error) { + if (err) { ngx_log_error(NGX_LOG_CRIT, ext->log, err, ngx_rename_file_n " \"%s\" to \"%s\" failed", src->data, to->data); } - ext->rename_error = err; + return NGX_ERROR; +} + + +ngx_int_t +ngx_copy_file(u_char *from, u_char *to, ngx_copy_file_t *cf) +{ + char *buf; + off_t size; + size_t len; + ssize_t n; + ngx_fd_t fd, nfd; + ngx_int_t rc; + ngx_file_info_t fi; + + rc = NGX_ERROR; + buf = NULL; + nfd = NGX_INVALID_FILE; + + fd = ngx_open_file(from, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); + + if (fd == NGX_INVALID_FILE) { + ngx_log_error(NGX_LOG_CRIT, cf->log, ngx_errno, + ngx_open_file_n " \"%s\" failed", from); + goto failed; + } + + if (cf->size != -1) { + size = cf->size; + + } else { + if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, + ngx_fd_info_n " \"%s\" failed", from); + + goto failed; + } + + size = ngx_file_size(&fi); + } + + len = cf->buf_size ? cf->buf_size : 65536; + + if ((off_t) len > size) { + len = (size_t) size; + } + + buf = ngx_alloc(len, cf->log); + if (buf == NULL) { + goto failed; + } + + nfd = ngx_open_file(to, NGX_FILE_WRONLY, NGX_FILE_CREATE_OR_OPEN, + cf->access); + + if (nfd == NGX_INVALID_FILE) { + ngx_log_error(NGX_LOG_CRIT, cf->log, ngx_errno, + ngx_open_file_n " \"%s\" failed", to); + goto failed; + } + + while (size > 0) { - return NGX_ERROR; + if ((off_t) len > size) { + len = (size_t) size; + } + + n = ngx_read_fd(fd, buf, len); + + if (n == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, + ngx_read_fd_n " \"%s\" failed", from); + goto failed; + } + + if ((size_t) n != len) { + ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, + ngx_read_fd_n " has read only %z of %uz from %s", + n, size, from); + goto failed; + } + + n = ngx_write_fd(nfd, buf, len); + + if (n == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, + ngx_write_fd_n " \"%s\" failed", to); + goto failed; + } + + if ((size_t) n != len) { + ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, + ngx_write_fd_n " has written only %z of %uz to %s", + n, size, to); + goto failed; + } + + size -= n; + } + + if (ngx_set_file_time(to, nfd, cf->time) != NGX_OK) { + ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, + ngx_set_file_time_n " \"%s\" failed", to); + goto failed; + } + + rc = NGX_OK; + +failed: + + if (nfd != NGX_INVALID_FILE) { + if (ngx_close_file(nfd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, + ngx_close_file_n " \"%s\" failed", to); + } + } + + if (fd != NGX_INVALID_FILE) { + if (ngx_close_file(fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, + ngx_close_file_n " \"%s\" failed", from); + } + } + + if (buf) { + ngx_free(buf); + } + + return rc; } diff --git a/src/core/ngx_file.h b/src/core/ngx_file.h --- a/src/core/ngx_file.h +++ b/src/core/ngx_file.h @@ -73,16 +73,25 @@ typedef struct { ngx_uint_t path_access; time_t time; ngx_fd_t fd; - ngx_err_t rename_error; unsigned create_path:1; unsigned delete_file:1; - unsigned log_rename_error:1; ngx_log_t *log; } ngx_ext_rename_file_t; +typedef struct { + off_t size; + size_t buf_size; + + ngx_uint_t access; + time_t time; + + ngx_log_t *log; +} ngx_copy_file_t; + + typedef struct ngx_tree_ctx_s ngx_tree_ctx_t; typedef ngx_int_t (*ngx_tree_init_handler_pt) (void *ctx, void *prev); @@ -117,9 +126,9 @@ ngx_int_t ngx_add_path(ngx_conf_t *cf, n ngx_int_t ngx_create_pathes(ngx_cycle_t *cycle, ngx_uid_t user); ngx_int_t ngx_ext_rename_file(ngx_str_t *src, ngx_str_t *to, ngx_ext_rename_file_t *ext); +ngx_int_t ngx_copy_file(u_char *from, u_char *to, ngx_copy_file_t *cf); ngx_int_t ngx_walk_tree(ngx_tree_ctx_t *ctx, ngx_str_t *tree); -void ngx_init_temp_number(void); ngx_atomic_uint_t ngx_next_temp_number(ngx_uint_t collision); char *ngx_conf_set_path_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -128,4 +137,8 @@ char *ngx_conf_merge_path_value(ngx_conf char *ngx_conf_set_access_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +extern ngx_atomic_t *ngx_temp_number; +extern ngx_atomic_int_t ngx_random_number; + + #endif /* _NGX_FILE_H_INCLUDED_ */ diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c --- a/src/event/ngx_event.c +++ b/src/event/ngx_event.c @@ -43,7 +43,7 @@ ngx_uint_t ngx_event_flags; ngx_event_actions_t ngx_event_actions; -ngx_atomic_t connection_counter = 1; +static ngx_atomic_t connection_counter = 1; ngx_atomic_t *ngx_connection_counter = &connection_counter; @@ -429,6 +429,7 @@ ngx_event_module_init(ngx_cycle_t *cycle u_char *shared; size_t size, cl; ngx_shm_t shm; + ngx_time_t *tp; ngx_core_conf_t *ccf; ngx_event_conf_t *ecf; @@ -492,7 +493,8 @@ ngx_event_module_init(ngx_cycle_t *cycle cl = 128; size = cl /* ngx_accept_mutex */ - + cl; /* ngx_connection_counter */ + + cl /* ngx_connection_counter */ + + cl; /* ngx_temp_number */ #if (NGX_STAT_STUB) @@ -526,23 +528,29 @@ ngx_event_module_init(ngx_cycle_t *cycle ngx_connection_counter = (ngx_atomic_t *) (shared + 1 * cl); -#if (NGX_STAT_STUB) - - ngx_stat_accepted = (ngx_atomic_t *) (shared + 2 * cl); - ngx_stat_handled = (ngx_atomic_t *) (shared + 3 * cl); - ngx_stat_requests = (ngx_atomic_t *) (shared + 4 * cl); - ngx_stat_active = (ngx_atomic_t *) (shared + 5 * cl); - ngx_stat_reading = (ngx_atomic_t *) (shared + 6 * cl); - ngx_stat_writing = (ngx_atomic_t *) (shared + 7 * cl); - -#endif - (void) ngx_atomic_cmp_set(ngx_connection_counter, 0, 1); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "counter: %p, %d", ngx_connection_counter, *ngx_connection_counter); + ngx_temp_number = (ngx_atomic_t *) (shared + 2 * cl); + + tp = ngx_timeofday(); + + ngx_random_number = (tp->msec << 16) + ngx_pid; + +#if (NGX_STAT_STUB) + + ngx_stat_accepted = (ngx_atomic_t *) (shared + 3 * cl); + ngx_stat_handled = (ngx_atomic_t *) (shared + 4 * cl); + ngx_stat_requests = (ngx_atomic_t *) (shared + 5 * cl); + ngx_stat_active = (ngx_atomic_t *) (shared + 6 * cl); + ngx_stat_reading = (ngx_atomic_t *) (shared + 7 * cl); + ngx_stat_writing = (ngx_atomic_t *) (shared + 8 * cl); + +#endif + return NGX_OK; } diff --git a/src/http/modules/ngx_http_dav_module.c b/src/http/modules/ngx_http_dav_module.c --- a/src/http/modules/ngx_http_dav_module.c +++ b/src/http/modules/ngx_http_dav_module.c @@ -53,8 +53,6 @@ static ngx_int_t ngx_http_dav_copy_dir_t ngx_str_t *path); static ngx_int_t ngx_http_dav_copy_tree_file(ngx_tree_ctx_t *ctx, ngx_str_t *path); -static ngx_int_t ngx_http_dav_copy_file(ngx_tree_ctx_t *ctx, u_char *from, - u_char *to); static ngx_int_t ngx_http_dav_depth(ngx_http_request_t *r, ngx_int_t dflt); static ngx_int_t ngx_http_dav_error(ngx_log_t *log, ngx_err_t err, @@ -216,6 +214,8 @@ ngx_http_dav_put_handler(ngx_http_reques ngx_http_map_uri_to_path(r, &path, &root, 0); + path.len--; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http put filename: \"%s\"", path.data); @@ -249,7 +249,6 @@ ngx_http_dav_put_handler(ngx_http_reques ext.time = -1; ext.create_path = dlcf->create_full_put_path; ext.delete_file = 1; - ext.log_rename_error = 1; ext.log = r->connection->log; if (r->headers_in.date) { @@ -520,6 +519,7 @@ ngx_http_dav_copy_move_handler(ngx_http_ ngx_uint_t overwrite, slash, dir; ngx_str_t path, uri; ngx_tree_ctx_t tree; + ngx_copy_file_t cf; ngx_file_info_t fi; ngx_table_elt_t *dest, *over; ngx_ext_rename_file_t ext; @@ -791,43 +791,24 @@ overwrite_done: ext.time = -1; ext.create_path = 1; ext.delete_file = 0; - ext.log_rename_error = 0; ext.log = r->connection->log; if (ngx_ext_rename_file(&path, ©.path, &ext) == NGX_OK) { return NGX_HTTP_NO_CONTENT; } - if (ext.rename_error != NGX_EXDEV) { - - if (ext.rename_error) { - ngx_log_error(NGX_LOG_CRIT, r->connection->log, - ext.rename_error, - ngx_rename_file_n " \"%s\" to \"%s\" failed", - path.data, copy.path.data); - } - - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } + return NGX_HTTP_INTERNAL_SERVER_ERROR; } dlcf = ngx_http_get_module_loc_conf(r, ngx_http_dav_module); - tree.size = ngx_file_size(&fi); - tree.mtime = ngx_file_mtime(&fi); - tree.access = dlcf->access; - tree.log = r->connection->log; + cf.size = ngx_file_size(&fi); + cf.buf_size = 0; + cf.access = dlcf->access; + cf.time = ngx_file_mtime(&fi); + cf.log = r->connection->log; - if (ngx_http_dav_copy_file(&tree, path.data, copy.path.data) == NGX_OK) - { - if (r->method == NGX_HTTP_MOVE) { - rc = ngx_http_dav_delete_path(r, &path, 0); - - if (rc != NGX_OK) { - return rc; - } - } - + if (ngx_copy_file(path.data, copy.path.data, &cf) == NGX_OK) { return NGX_HTTP_NO_CONTENT; } } @@ -941,6 +922,7 @@ ngx_http_dav_copy_tree_file(ngx_tree_ctx { u_char *p, *file; size_t len; + ngx_copy_file_t cf; ngx_http_dav_copy_ctx_t *copy; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0, @@ -961,7 +943,13 @@ ngx_http_dav_copy_tree_file(ngx_tree_ctx ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0, "http copy file to: \"%s\"", file); - (void) ngx_http_dav_copy_file(ctx, path->data, file); + cf.size = ctx->size; + cf.buf_size = 0; + cf.access = ctx->access; + cf.time = ctx->mtime; + cf.log = ctx->log; + + (void) ngx_copy_file(path->data, file, &cf); ngx_free(file); @@ -970,75 +958,6 @@ ngx_http_dav_copy_tree_file(ngx_tree_ctx static ngx_int_t -ngx_http_dav_copy_file(ngx_tree_ctx_t *ctx, u_char *from, u_char *to) -{ - off_t size; - ssize_t n; - ngx_fd_t fd, cfd; - ngx_int_t rc; - u_char buf[NGX_HTTP_DAV_COPY_BLOCK]; - - fd = ngx_open_file(from, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); - - if (fd == NGX_INVALID_FILE) { - (void) ngx_http_dav_error(ctx->log, ngx_errno, 0, ngx_open_file_n, - from); - return NGX_ERROR; - } - - cfd = ngx_open_file(to, NGX_FILE_WRONLY, NGX_FILE_CREATE_OR_OPEN, - ctx->access); - - rc = NGX_ERROR; - - if (cfd == NGX_INVALID_FILE) { - (void) ngx_http_dav_error(ctx->log, ngx_errno, 0, ngx_open_file_n, to); - goto failed; - } - - for (size = ctx->size; size > 0; size -= n) { - - n = ngx_read_fd(fd, buf, NGX_HTTP_DAV_COPY_BLOCK); - - if (n == NGX_FILE_ERROR) { - ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno, - ngx_read_fd_n " \"%s\" failed", from); - goto failed; - } - - if (ngx_write_fd(cfd, buf, n) == NGX_FILE_ERROR) { - ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno, - ngx_write_fd_n " \"%s\" failed", to); - goto failed; - } - } - - if (ngx_set_file_time(to, cfd, ctx->mtime) != NGX_OK) { - ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno, - ngx_set_file_time_n " \"%s\" failed", to); - goto failed; - } - - if (ngx_close_file(cfd) == NGX_FILE_ERROR) { - ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno, - ngx_close_file_n " \"%s\" failed", to); - goto failed; - } - - rc = NGX_OK; - -failed: - - if (ngx_close_file(fd) == NGX_FILE_ERROR) { - ngx_log_error(NGX_LOG_ALERT, ctx->log, ngx_errno, - ngx_close_file_n " \"%s\" failed", from); - } - - return rc; -} - - -static ngx_int_t ngx_http_dav_depth(ngx_http_request_t *r, ngx_int_t dflt) { ngx_table_elt_t *depth; diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -651,7 +651,6 @@ ngx_http_file_cache_update(ngx_http_requ ext.time = -1; ext.create_path = 1; ext.delete_file = 1; - ext.log_rename_error = 1; ext.log = r->connection->log; rc = ngx_ext_rename_file(&tf->file.name, &c->file.name, &ext); diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -2640,7 +2640,6 @@ ngx_http_upstream_store(ngx_http_request ext.time = -1; ext.create_path = 1; ext.delete_file = 1; - ext.log_rename_error = 1; ext.log = r->connection->log; if (u->headers_in.last_modified) { @@ -2667,6 +2666,8 @@ ngx_http_upstream_store(ngx_http_request } } + path.len--; + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "upstream stores \"%s\" to \"%s\"", tf->file.name.data, path.data); diff --git a/src/os/unix/ngx_errno.h b/src/os/unix/ngx_errno.h --- a/src/os/unix/ngx_errno.h +++ b/src/os/unix/ngx_errno.h @@ -16,6 +16,7 @@ typedef int ngx_err_t; #define NGX_EPERM EPERM #define NGX_ENOENT ENOENT +#define NGX_ENOPATH ENOENT #define NGX_ESRCH ESRCH #define NGX_EINTR EINTR #define NGX_ECHILD ECHILD diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c --- a/src/os/unix/ngx_process_cycle.c +++ b/src/os/unix/ngx_process_cycle.c @@ -281,8 +281,6 @@ ngx_single_process_cycle(ngx_cycle_t *cy { ngx_uint_t i; - ngx_init_temp_number(); - for (i = 0; ngx_modules[i]; i++) { if (ngx_modules[i]->init_process) { if (ngx_modules[i]->init_process(cycle) == NGX_ERROR) { @@ -933,8 +931,6 @@ ngx_worker_process_init(ngx_cycle_t *cyc "sigprocmask() failed"); } - ngx_init_temp_number(); - /* * disable deleting previous events for the listening sockets because * in the worker processes there are no events at all at this point diff --git a/src/os/win32/ngx_errno.h b/src/os/win32/ngx_errno.h --- a/src/os/win32/ngx_errno.h +++ b/src/os/win32/ngx_errno.h @@ -21,6 +21,7 @@ typedef DWORD ngx_e #define NGX_EPERM ERROR_ACCESS_DENIED #define NGX_ENOENT ERROR_FILE_NOT_FOUND +#define NGX_ENOPATH ERROR_PATH_NOT_FOUND #define NGX_ENOMEM ERROR_NOT_ENOUGH_MEMORY #define NGX_EACCES ERROR_ACCESS_DENIED /* it's seems that ERROR_FILE_EXISTS is not appropriate error code */ diff --git a/src/os/win32/ngx_process_cycle.c b/src/os/win32/ngx_process_cycle.c --- a/src/os/win32/ngx_process_cycle.c +++ b/src/os/win32/ngx_process_cycle.c @@ -793,8 +793,6 @@ ngx_worker_thread(void *data) cycle = (ngx_cycle_t *) ngx_cycle; - ngx_init_temp_number(); - for (n = 0; ngx_modules[n]; n++) { if (ngx_modules[n]->init_process) { if (ngx_modules[n]->init_process(cycle) == NGX_ERROR) { @@ -1024,8 +1022,6 @@ ngx_single_process_cycle(ngx_cycle_t *cy ngx_int_t i; ngx_tid_t tid; - ngx_init_temp_number(); - for (i = 0; ngx_modules[i]; i++) { if (ngx_modules[i]->init_process) { if (ngx_modules[i]->init_process(cycle) == NGX_ERROR) {