Mercurial > hg > nginx

comparison src/stream/ngx_stream_upstream_round_robin.c @ 6115:61d7ae76647d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Stream: port from NGINX+.
author Ruslan Ermilov <ru@nginx.com>
date Mon, 20 Apr 2015 13:05:11 +0300
parents
children cca856715722
comparison
equal deleted inserted replaced
6114:4a640716f4e2 6115:61d7ae76647d
1
2 /*
3 * Copyright (C) Igor Sysoev
4 * Copyright (C) Nginx, Inc.
5 */
6
7
8 #include <ngx_config.h>
9 #include <ngx_core.h>
10 #include <ngx_stream.h>
11
12
13 #define ngx_stream_upstream_tries(p) ((p)->number \
14 + ((p)->next ? (p)->next->number : 0))
15
16
17 static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_get_peer(
18 ngx_stream_upstream_rr_peer_data_t *rrp);
19
20 #if (NGX_STREAM_SSL)
21
22 static ngx_int_t ngx_stream_upstream_set_round_robin_peer_session(
23 ngx_peer_connection_t *pc, void *data);
24 static void ngx_stream_upstream_save_round_robin_peer_session(
25 ngx_peer_connection_t *pc, void *data);
26
27 #endif
28
29
30 ngx_int_t
31 ngx_stream_upstream_init_round_robin(ngx_conf_t *cf,
32 ngx_stream_upstream_srv_conf_t *us)
33 {
34 ngx_url_t u;
35 ngx_uint_t i, j, n, w;
36 ngx_stream_upstream_server_t *server;
37 ngx_stream_upstream_rr_peer_t *peer, **peerp;
38 ngx_stream_upstream_rr_peers_t *peers, *backup;
39
40 us->peer.init = ngx_stream_upstream_init_round_robin_peer;
41
42 if (us->servers) {
43 server = us->servers->elts;
44
45 n = 0;
46 w = 0;
47
48 for (i = 0; i < us->servers->nelts; i++) {
49 if (server[i].backup) {
50 continue;
51 }
52
53 n += server[i].naddrs;
54 w += server[i].naddrs * server[i].weight;
55 }
56
57 if (n == 0) {
58 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
59 "no servers in upstream \"%V\" in %s:%ui",
60 &us->host, us->file_name, us->line);
61 return NGX_ERROR;
62 }
63
64 peers = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peers_t));
65 if (peers == NULL) {
66 return NGX_ERROR;
67 }
68
69 peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) * n);
70 if (peer == NULL) {
71 return NGX_ERROR;
72 }
73
74 peers->single = (n == 1);
75 peers->number = n;
76 peers->weighted = (w != n);
77 peers->total_weight = w;
78 peers->name = &us->host;
79
80 n = 0;
81 peerp = &peers->peer;
82
83 for (i = 0; i < us->servers->nelts; i++) {
84 if (server[i].backup) {
85 continue;
86 }
87
88 for (j = 0; j < server[i].naddrs; j++) {
89 peer[n].sockaddr = server[i].addrs[j].sockaddr;
90 peer[n].socklen = server[i].addrs[j].socklen;
91 peer[n].name = server[i].addrs[j].name;
92 peer[n].weight = server[i].weight;
93 peer[n].effective_weight = server[i].weight;
94 peer[n].current_weight = 0;
95 peer[n].max_fails = server[i].max_fails;
96 peer[n].fail_timeout = server[i].fail_timeout;
97 peer[n].down = server[i].down;
98 peer[n].server = server[i].name;
99
100 *peerp = &peer[n];
101 peerp = &peer[n].next;
102 n++;
103 }
104 }
105
106 us->peer.data = peers;
107
108 /* backup servers */
109
110 n = 0;
111 w = 0;
112
113 for (i = 0; i < us->servers->nelts; i++) {
114 if (!server[i].backup) {
115 continue;
116 }
117
118 n += server[i].naddrs;
119 w += server[i].naddrs * server[i].weight;
120 }
121
122 if (n == 0) {
123 return NGX_OK;
124 }
125
126 backup = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peers_t));
127 if (backup == NULL) {
128 return NGX_ERROR;
129 }
130
131 peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) * n);
132 if (peer == NULL) {
133 return NGX_ERROR;
134 }
135
136 peers->single = 0;
137 backup->single = 0;
138 backup->number = n;
139 backup->weighted = (w != n);
140 backup->total_weight = w;
141 backup->name = &us->host;
142
143 n = 0;
144 peerp = &backup->peer;
145
146 for (i = 0; i < us->servers->nelts; i++) {
147 if (!server[i].backup) {
148 continue;
149 }
150
151 for (j = 0; j < server[i].naddrs; j++) {
152 peer[n].sockaddr = server[i].addrs[j].sockaddr;
153 peer[n].socklen = server[i].addrs[j].socklen;
154 peer[n].name = server[i].addrs[j].name;
155 peer[n].weight = server[i].weight;
156 peer[n].effective_weight = server[i].weight;
157 peer[n].current_weight = 0;
158 peer[n].max_fails = server[i].max_fails;
159 peer[n].fail_timeout = server[i].fail_timeout;
160 peer[n].down = server[i].down;
161 peer[n].server = server[i].name;
162
163 *peerp = &peer[n];
164 peerp = &peer[n].next;
165 n++;
166 }
167 }
168
169 peers->next = backup;
170
171 return NGX_OK;
172 }
173
174
175 /* an upstream implicitly defined by proxy_pass, etc. */
176
177 if (us->port == 0) {
178 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
179 "no port in upstream \"%V\" in %s:%ui",
180 &us->host, us->file_name, us->line);
181 return NGX_ERROR;
182 }
183
184 ngx_memzero(&u, sizeof(ngx_url_t));
185
186 u.host = us->host;
187 u.port = us->port;
188
189 if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
190 if (u.err) {
191 ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
192 "%s in upstream \"%V\" in %s:%ui",
193 u.err, &us->host, us->file_name, us->line);
194 }
195
196 return NGX_ERROR;
197 }
198
199 n = u.naddrs;
200
201 peers = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peers_t));
202 if (peers == NULL) {
203 return NGX_ERROR;
204 }
205
206 peer = ngx_pcalloc(cf->pool, sizeof(ngx_stream_upstream_rr_peer_t) * n);
207 if (peer == NULL) {
208 return NGX_ERROR;
209 }
210
211 peers->single = (n == 1);
212 peers->number = n;
213 peers->weighted = 0;
214 peers->total_weight = n;
215 peers->name = &us->host;
216
217 peerp = &peers->peer;
218
219 for (i = 0; i < u.naddrs; i++) {
220 peer[i].sockaddr = u.addrs[i].sockaddr;
221 peer[i].socklen = u.addrs[i].socklen;
222 peer[i].name = u.addrs[i].name;
223 peer[i].weight = 1;
224 peer[i].effective_weight = 1;
225 peer[i].current_weight = 0;
226 peer[i].max_fails = 1;
227 peer[i].fail_timeout = 10;
228 *peerp = &peer[i];
229 peerp = &peer[i].next;
230 }
231
232 us->peer.data = peers;
233
234 /* implicitly defined upstream has no backup servers */
235
236 return NGX_OK;
237 }
238
239
240 ngx_int_t
241 ngx_stream_upstream_init_round_robin_peer(ngx_stream_session_t *s,
242 ngx_stream_upstream_srv_conf_t *us)
243 {
244 ngx_uint_t n;
245 ngx_stream_upstream_rr_peer_data_t *rrp;
246
247 rrp = s->upstream->peer.data;
248
249 if (rrp == NULL) {
250 rrp = ngx_palloc(s->connection->pool,
251 sizeof(ngx_stream_upstream_rr_peer_data_t));
252 if (rrp == NULL) {
253 return NGX_ERROR;
254 }
255
256 s->upstream->peer.data = rrp;
257 }
258
259 rrp->peers = us->peer.data;
260 rrp->current = NULL;
261
262 n = rrp->peers->number;
263
264 if (rrp->peers->next && rrp->peers->next->number > n) {
265 n = rrp->peers->next->number;
266 }
267
268 if (n <= 8 * sizeof(uintptr_t)) {
269 rrp->tried = &rrp->data;
270 rrp->data = 0;
271
272 } else {
273 n = (n + (8 * sizeof(uintptr_t) - 1)) / (8 * sizeof(uintptr_t));
274
275 rrp->tried = ngx_pcalloc(s->connection->pool, n * sizeof(uintptr_t));
276 if (rrp->tried == NULL) {
277 return NGX_ERROR;
278 }
279 }
280
281 s->upstream->peer.get = ngx_stream_upstream_get_round_robin_peer;
282 s->upstream->peer.free = ngx_stream_upstream_free_round_robin_peer;
283 s->upstream->peer.tries = ngx_stream_upstream_tries(rrp->peers);
284 #if (NGX_STREAM_SSL)
285 s->upstream->peer.set_session =
286 ngx_stream_upstream_set_round_robin_peer_session;
287 s->upstream->peer.save_session =
288 ngx_stream_upstream_save_round_robin_peer_session;
289 #endif
290
291 return NGX_OK;
292 }
293
294
295 ngx_int_t
296 ngx_stream_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
297 {
298 ngx_stream_upstream_rr_peer_data_t *rrp = data;
299
300 ngx_int_t rc;
301 ngx_uint_t i, n;
302 ngx_stream_upstream_rr_peer_t *peer;
303 ngx_stream_upstream_rr_peers_t *peers;
304
305 ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
306 "get rr peer, try: %ui", pc->tries);
307
308 pc->connection = NULL;
309
310 peers = rrp->peers;
311 ngx_stream_upstream_rr_peers_wlock(peers);
312
313 if (peers->single) {
314 peer = peers->peer;
315
316 if (peer->down) {
317 goto failed;
318 }
319
320 rrp->current = peer;
321
322 } else {
323
324 /* there are several peers */
325
326 peer = ngx_stream_upstream_get_peer(rrp);
327
328 if (peer == NULL) {
329 goto failed;
330 }
331
332 ngx_log_debug2(NGX_LOG_DEBUG_STREAM, pc->log, 0,
333 "get rr peer, current: %p %i",
334 peer, peer->current_weight);
335 }
336
337 pc->sockaddr = peer->sockaddr;
338 pc->socklen = peer->socklen;
339 pc->name = &peer->name;
340
341 peer->conns++;
342
343 ngx_stream_upstream_rr_peers_unlock(peers);
344
345 return NGX_OK;
346
347 failed:
348
349 if (peers->next) {
350
351 ngx_log_debug0(NGX_LOG_DEBUG_STREAM, pc->log, 0, "backup servers");
352
353 rrp->peers = peers->next;
354
355 n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
356 / (8 * sizeof(uintptr_t));
357
358 for (i = 0; i < n; i++) {
359 rrp->tried[i] = 0;
360 }
361
362 ngx_stream_upstream_rr_peers_unlock(peers);
363
364 rc = ngx_stream_upstream_get_round_robin_peer(pc, rrp);
365
366 if (rc != NGX_BUSY) {
367 return rc;
368 }
369
370 ngx_stream_upstream_rr_peers_wlock(peers);
371 }
372
373 /* all peers failed, mark them as live for quick recovery */
374
375 for (peer = peers->peer; peer; peer = peer->next) {
376 peer->fails = 0;
377 }
378
379 ngx_stream_upstream_rr_peers_unlock(peers);
380
381 pc->name = peers->name;
382
383 return NGX_BUSY;
384 }
385
386
387 static ngx_stream_upstream_rr_peer_t *
388 ngx_stream_upstream_get_peer(ngx_stream_upstream_rr_peer_data_t *rrp)
389 {
390 time_t now;
391 uintptr_t m;
392 ngx_int_t total;
393 ngx_uint_t i, n, p;
394 ngx_stream_upstream_rr_peer_t *peer, *best;
395
396 now = ngx_time();
397
398 best = NULL;
399 total = 0;
400
401 #if (NGX_SUPPRESS_WARN)
402 p = 0;
403 #endif
404
405 for (peer = rrp->peers->peer, i = 0;
406 peer;
407 peer = peer->next, i++)
408 {
409
410 n = i / (8 * sizeof(uintptr_t));
411 m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
412
413 if (rrp->tried[n] & m) {
414 continue;
415 }
416
417 if (peer->down) {
418 continue;
419 }
420
421 if (peer->max_fails
422 && peer->fails >= peer->max_fails
423 && now - peer->checked <= peer->fail_timeout)
424 {
425 continue;
426 }
427
428 peer->current_weight += peer->effective_weight;
429 total += peer->effective_weight;
430
431 if (peer->effective_weight < peer->weight) {
432 peer->effective_weight++;
433 }
434
435 if (best == NULL || peer->current_weight > best->current_weight) {
436 best = peer;
437 p = i;
438 }
439 }
440
441 if (best == NULL) {
442 return NULL;
443 }
444
445 rrp->current = best;
446
447 n = p / (8 * sizeof(uintptr_t));
448 m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
449
450 rrp->tried[n] |= m;
451
452 best->current_weight -= total;
453
454 if (now - best->checked > best->fail_timeout) {
455 best->checked = now;
456 }
457
458 return best;
459 }
460
461
462 void
463 ngx_stream_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
464 ngx_uint_t state)
465 {
466 ngx_stream_upstream_rr_peer_data_t *rrp = data;
467
468 time_t now;
469 ngx_stream_upstream_rr_peer_t *peer;
470
471 ngx_log_debug2(NGX_LOG_DEBUG_STREAM, pc->log, 0,
472 "free rr peer %ui %ui", pc->tries, state);
473
474 peer = rrp->current;
475
476 ngx_stream_upstream_rr_peers_rlock(rrp->peers);
477 ngx_stream_upstream_rr_peer_lock(rrp->peers, peer);
478
479 if (rrp->peers->single) {
480 peer->conns--;
481
482 ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer);
483 ngx_stream_upstream_rr_peers_unlock(rrp->peers);
484
485 pc->tries = 0;
486 return;
487 }
488
489 if (state & NGX_PEER_FAILED) {
490 now = ngx_time();
491
492 peer->fails++;
493 peer->accessed = now;
494 peer->checked = now;
495
496 if (peer->max_fails) {
497 peer->effective_weight -= peer->weight / peer->max_fails;
498 }
499
500 ngx_log_debug2(NGX_LOG_DEBUG_STREAM, pc->log, 0,
501 "free rr peer failed: %p %i",
502 peer, peer->effective_weight);
503
504 if (peer->effective_weight < 0) {
505 peer->effective_weight = 0;
506 }
507
508 } else {
509
510 /* mark peer live if check passed */
511
512 if (peer->accessed < peer->checked) {
513 peer->fails = 0;
514 }
515 }
516
517 peer->conns--;
518
519 ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer);
520 ngx_stream_upstream_rr_peers_unlock(rrp->peers);
521
522 if (pc->tries) {
523 pc->tries--;
524 }
525 }
526
527
528 #if (NGX_STREAM_SSL)
529
530 static ngx_int_t
531 ngx_stream_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
532 void *data)
533 {
534 ngx_stream_upstream_rr_peer_data_t *rrp = data;
535
536 ngx_int_t rc;
537 ngx_ssl_session_t *ssl_session;
538 ngx_stream_upstream_rr_peer_t *peer;
539 #if (NGX_STREAM_UPSTREAM_ZONE)
540 int len;
541 #if OPENSSL_VERSION_NUMBER >= 0x0090707fL
542 const
543 #endif
544 u_char *p;
545 ngx_stream_upstream_rr_peers_t *peers;
546 u_char buf[NGX_SSL_MAX_SESSION_SIZE];
547 #endif
548
549 peer = rrp->current;
550
551 #if (NGX_STREAM_UPSTREAM_ZONE)
552 peers = rrp->peers;
553
554 if (peers->shpool) {
555 ngx_stream_upstream_rr_peers_rlock(peers);
556 ngx_stream_upstream_rr_peer_lock(peers, peer);
557
558 if (peer->ssl_session == NULL) {
559 ngx_stream_upstream_rr_peer_unlock(peers, peer);
560 ngx_stream_upstream_rr_peers_unlock(peers);
561 return NGX_OK;
562 }
563
564 len = peer->ssl_session_len;
565
566 ngx_memcpy(buf, peer->ssl_session, len);
567
568 ngx_stream_upstream_rr_peer_unlock(peers, peer);
569 ngx_stream_upstream_rr_peers_unlock(peers);
570
571 p = buf;
572 ssl_session = d2i_SSL_SESSION(NULL, &p, len);
573
574 rc = ngx_ssl_set_session(pc->connection, ssl_session);
575
576 ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
577 "set session: %p", ssl_session);
578
579 ngx_ssl_free_session(ssl_session);
580
581 return rc;
582 }
583 #endif
584
585 ssl_session = peer->ssl_session;
586
587 rc = ngx_ssl_set_session(pc->connection, ssl_session);
588
589 ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
590 "set session: %p", ssl_session);
591
592 return rc;
593 }
594
595
596 static void
597 ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
598 void *data)
599 {
600 ngx_stream_upstream_rr_peer_data_t *rrp = data;
601
602 ngx_ssl_session_t *old_ssl_session, *ssl_session;
603 ngx_stream_upstream_rr_peer_t *peer;
604 #if (NGX_STREAM_UPSTREAM_ZONE)
605 int len;
606 u_char *p;
607 ngx_stream_upstream_rr_peers_t *peers;
608 u_char buf[NGX_SSL_MAX_SESSION_SIZE];
609 #endif
610
611 #if (NGX_STREAM_UPSTREAM_ZONE)
612 peers = rrp->peers;
613
614 if (peers->shpool) {
615
616 ssl_session = SSL_get0_session(pc->connection->ssl->connection);
617
618 if (ssl_session == NULL) {
619 return;
620 }
621
622 ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
623 "save session: %p", ssl_session);
624
625 len = i2d_SSL_SESSION(ssl_session, NULL);
626
627 /* do not cache too big session */
628
629 if (len > NGX_SSL_MAX_SESSION_SIZE) {
630 return;
631 }
632
633 p = buf;
634 (void) i2d_SSL_SESSION(ssl_session, &p);
635
636 peer = rrp->current;
637
638 ngx_stream_upstream_rr_peers_rlock(peers);
639 ngx_stream_upstream_rr_peer_lock(peers, peer);
640
641 if (len > peer->ssl_session_len) {
642 ngx_shmtx_lock(&peers->shpool->mutex);
643
644 if (peer->ssl_session) {
645 ngx_slab_free_locked(peers->shpool, peer->ssl_session);
646 }
647
648 peer->ssl_session = ngx_slab_alloc_locked(peers->shpool, len);
649
650 ngx_shmtx_unlock(&peers->shpool->mutex);
651
652 if (peer->ssl_session == NULL) {
653 peer->ssl_session_len = 0;
654
655 ngx_stream_upstream_rr_peer_unlock(peers, peer);
656 ngx_stream_upstream_rr_peers_unlock(peers);
657 return;
658 }
659
660 peer->ssl_session_len = len;
661 }
662
663 ngx_memcpy(peer->ssl_session, buf, len);
664
665 ngx_stream_upstream_rr_peer_unlock(peers, peer);
666 ngx_stream_upstream_rr_peers_unlock(peers);
667
668 return;
669 }
670 #endif
671
672 ssl_session = ngx_ssl_get_session(pc->connection);
673
674 if (ssl_session == NULL) {
675 return;
676 }
677
678 ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
679 "save session: %p", ssl_session);
680
681 peer = rrp->current;
682
683 old_ssl_session = peer->ssl_session;
684 peer->ssl_session = ssl_session;
685
686 if (old_ssl_session) {
687
688 ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
689 "old session: %p", old_ssl_session);
690
691 /* TODO: may block */
692
693 ngx_ssl_free_session(old_ssl_session);
694 }
695 }
696
697 #endif