comparison src/event/ngx_event_quic.c @ 8364:eee307399229 quic

QUIC basic congestion control.
author Roman Arutyunyan <arut@nginx.com>
date Tue, 28 Apr 2020 16:42:43 +0300
parents d3395396ce51
children fab75acb1f72
comparison
equal deleted inserted replaced
8363:d3395396ce51 8364:eee307399229
51 uint64_t total_received; 51 uint64_t total_received;
52 uint64_t max_data; 52 uint64_t max_data;
53 } ngx_quic_streams_t; 53 } ngx_quic_streams_t;
54 54
55 55
56 typedef struct {
57 size_t in_flight;
58 size_t window;
59 size_t ssthresh;
60 ngx_msec_t recovery_start;
61 } ngx_quic_congestion_t;
62
63
56 /* 64 /*
57 * 12.3. Packet Numbers 65 * 12.3. Packet Numbers
58 * 66 *
59 * Conceptually, a packet number space is the context in which a packet 67 * Conceptually, a packet number space is the context in which a packet
60 * can be processed and acknowledged. Initial packets can only be sent 68 * can be processed and acknowledged. Initial packets can only be sent
101 #if (NGX_DEBUG) 109 #if (NGX_DEBUG)
102 ngx_uint_t nframes; 110 ngx_uint_t nframes;
103 #endif 111 #endif
104 112
105 ngx_quic_streams_t streams; 113 ngx_quic_streams_t streams;
114 ngx_quic_congestion_t congestion;
106 ngx_uint_t max_data; 115 ngx_uint_t max_data;
107 116
108 uint64_t cur_streams; 117 uint64_t cur_streams;
109 uint64_t max_streams; 118 uint64_t max_streams;
110 119
169 178
170 static ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c, 179 static ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c,
171 ngx_quic_header_t *pkt, ngx_quic_ack_frame_t *f); 180 ngx_quic_header_t *pkt, ngx_quic_ack_frame_t *f);
172 static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c, 181 static ngx_int_t ngx_quic_handle_ack_frame_range(ngx_connection_t *c,
173 ngx_quic_send_ctx_t *ctx, uint64_t min, uint64_t max); 182 ngx_quic_send_ctx_t *ctx, uint64_t min, uint64_t max);
183 static void ngx_quic_handle_stream_ack(ngx_connection_t *c,
184 ngx_quic_frame_t *f);
174 185
175 static ngx_int_t ngx_quic_handle_ordered_frame(ngx_connection_t *c, 186 static ngx_int_t ngx_quic_handle_ordered_frame(ngx_connection_t *c,
176 ngx_quic_frames_stream_t *fs, ngx_quic_frame_t *frame, 187 ngx_quic_frames_stream_t *fs, ngx_quic_frame_t *frame,
177 ngx_quic_frame_handler_pt handler); 188 ngx_quic_frame_handler_pt handler);
178 static ngx_int_t ngx_quic_adjust_frame_offset(ngx_connection_t *c, 189 static ngx_int_t ngx_quic_adjust_frame_offset(ngx_connection_t *c,
225 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c, 236 static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c,
226 ngx_chain_t *in, off_t limit); 237 ngx_chain_t *in, off_t limit);
227 static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c, size_t size); 238 static ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c, size_t size);
228 static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame); 239 static void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame);
229 240
241 static void ngx_quic_congestion_ack(ngx_connection_t *c,
242 ngx_quic_frame_t *frame);
243 static void ngx_quic_congestion_lost(ngx_connection_t *c, ngx_msec_t sent);
244
230 245
231 static SSL_QUIC_METHOD quic_method = { 246 static SSL_QUIC_METHOD quic_method = {
232 #if BORINGSSL_API_VERSION >= 10 247 #if BORINGSSL_API_VERSION >= 10
233 ngx_quic_set_read_secret, 248 ngx_quic_set_read_secret,
234 ngx_quic_set_write_secret, 249 ngx_quic_set_write_secret,
583 ctp->max_packet_size = NGX_QUIC_DEFAULT_MAX_PACKET_SIZE; 598 ctp->max_packet_size = NGX_QUIC_DEFAULT_MAX_PACKET_SIZE;
584 ctp->ack_delay_exponent = NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT; 599 ctp->ack_delay_exponent = NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT;
585 ctp->max_ack_delay = NGX_QUIC_DEFAULT_MAX_ACK_DELAY; 600 ctp->max_ack_delay = NGX_QUIC_DEFAULT_MAX_ACK_DELAY;
586 601
587 qc->streams.max_data = qc->tp.initial_max_data; 602 qc->streams.max_data = qc->tp.initial_max_data;
603
604 qc->congestion.window = ngx_min(10 * qc->tp.max_packet_size,
605 ngx_max(2 * qc->tp.max_packet_size, 14720));
606 qc->congestion.ssthresh = NGX_MAX_SIZE_T_VALUE;
607 qc->congestion.recovery_start = ngx_current_msec;
588 608
589 qc->dcid.len = pkt->dcid.len; 609 qc->dcid.len = pkt->dcid.len;
590 qc->dcid.data = ngx_pnalloc(c->pool, pkt->dcid.len); 610 qc->dcid.data = ngx_pnalloc(c->pool, pkt->dcid.len);
591 if (qc->dcid.data == NULL) { 611 if (qc->dcid.data == NULL) {
592 return NGX_ERROR; 612 return NGX_ERROR;
1608 1628
1609 static ngx_int_t 1629 static ngx_int_t
1610 ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, 1630 ngx_quic_handle_ack_frame_range(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx,
1611 uint64_t min, uint64_t max) 1631 uint64_t min, uint64_t max)
1612 { 1632 {
1613 ngx_uint_t found; 1633 ngx_uint_t found;
1614 ngx_queue_t *q; 1634 ngx_queue_t *q;
1615 ngx_quic_frame_t *f; 1635 ngx_quic_frame_t *f;
1636 ngx_quic_connection_t *qc;
1637
1638 qc = c->quic;
1616 1639
1617 found = 0; 1640 found = 0;
1618 1641
1619 q = ngx_queue_head(&ctx->sent); 1642 q = ngx_queue_head(&ctx->sent);
1620 1643
1621 while (q != ngx_queue_sentinel(&ctx->sent)) { 1644 while (q != ngx_queue_sentinel(&ctx->sent)) {
1622 1645
1623 f = ngx_queue_data(q, ngx_quic_frame_t, queue); 1646 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
1624 1647
1625 if (f->pnum >= min && f->pnum <= max) { 1648 if (f->pnum >= min && f->pnum <= max) {
1649 ngx_quic_congestion_ack(c, f);
1650
1651 ngx_quic_handle_stream_ack(c, f);
1652
1626 q = ngx_queue_next(q); 1653 q = ngx_queue_next(q);
1627 ngx_queue_remove(&f->queue); 1654 ngx_queue_remove(&f->queue);
1628 ngx_quic_free_frame(c, f); 1655 ngx_quic_free_frame(c, f);
1629 found = 1; 1656 found = 1;
1630 1657
1644 "quic ACK for the packet not in sent queue "); 1671 "quic ACK for the packet not in sent queue ");
1645 // TODO: handle error properly: PROTOCOL VIOLATION? 1672 // TODO: handle error properly: PROTOCOL VIOLATION?
1646 return NGX_ERROR; 1673 return NGX_ERROR;
1647 } 1674 }
1648 1675
1676 if (!qc->push.timer_set) {
1677 ngx_post_event(&qc->push, &ngx_posted_events);
1678 }
1679
1649 return NGX_OK; 1680 return NGX_OK;
1681 }
1682
1683
1684 static void
1685 ngx_quic_handle_stream_ack(ngx_connection_t *c, ngx_quic_frame_t *f)
1686 {
1687 uint64_t sent, unacked;
1688 ngx_event_t *wev;
1689 ngx_quic_stream_t *sn;
1690 ngx_quic_connection_t *qc;
1691
1692 if (f->type < NGX_QUIC_FT_STREAM0 || f->type > NGX_QUIC_FT_STREAM7) {
1693 return;
1694 }
1695
1696 qc = c->quic;
1697
1698 sn = ngx_quic_find_stream(&qc->streams.tree, f->u.stream.stream_id);
1699 if (sn == NULL) {
1700 return;
1701 }
1702
1703 wev = sn->c->write;
1704 sent = sn->c->sent;
1705 unacked = sent - sn->acked;
1706
1707 if (unacked >= NGX_QUIC_STREAM_BUFSIZE && wev->active) {
1708 wev->ready = 1;
1709 ngx_post_event(wev, &ngx_posted_events);
1710 }
1711
1712 sn->acked += f->u.stream.length;
1713
1714 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, sn->c->log, 0,
1715 "quic stream ack %uL acked:%uL, unacked:%uL",
1716 f->u.stream.length, sn->acked, sent - sn->acked);
1650 } 1717 }
1651 1718
1652 1719
1653 static ngx_int_t 1720 static ngx_int_t
1654 ngx_quic_handle_ordered_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs, 1721 ngx_quic_handle_ordered_frame(ngx_connection_t *c, ngx_quic_frames_stream_t *fs,
2261 static ngx_int_t 2328 static ngx_int_t
2262 ngx_quic_output_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx) 2329 ngx_quic_output_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx)
2263 { 2330 {
2264 size_t len, hlen, n; 2331 size_t len, hlen, n;
2265 ngx_int_t rc; 2332 ngx_int_t rc;
2333 ngx_uint_t need_ack;
2266 ngx_queue_t *q, range; 2334 ngx_queue_t *q, range;
2267 ngx_quic_frame_t *f; 2335 ngx_quic_frame_t *f;
2336 ngx_quic_congestion_t *cg;
2268 ngx_quic_connection_t *qc; 2337 ngx_quic_connection_t *qc;
2269 2338
2270 qc = c->quic; 2339 qc = c->quic;
2340 cg = &qc->congestion;
2271 2341
2272 if (ngx_queue_empty(&ctx->frames)) { 2342 if (ngx_queue_empty(&ctx->frames)) {
2273 return NGX_OK; 2343 return NGX_OK;
2274 } 2344 }
2275 2345
2281 : NGX_QUIC_MAX_LONG_HEADER; 2351 : NGX_QUIC_MAX_LONG_HEADER;
2282 hlen += EVP_GCM_TLS_TAG_LEN; 2352 hlen += EVP_GCM_TLS_TAG_LEN;
2283 2353
2284 do { 2354 do {
2285 len = 0; 2355 len = 0;
2356 need_ack = 0;
2286 ngx_queue_init(&range); 2357 ngx_queue_init(&range);
2287 2358
2288 do { 2359 do {
2289 /* process group of frames that fits into packet */ 2360 /* process group of frames that fits into packet */
2290 f = ngx_queue_data(q, ngx_quic_frame_t, queue); 2361 f = ngx_queue_data(q, ngx_quic_frame_t, queue);
2293 2364
2294 if (len && hlen + len + n > qc->ctp.max_packet_size) { 2365 if (len && hlen + len + n > qc->ctp.max_packet_size) {
2295 break; 2366 break;
2296 } 2367 }
2297 2368
2369 if (f->need_ack) {
2370 need_ack = 1;
2371 }
2372
2373 if (need_ack && cg->in_flight + len + n > cg->window) {
2374 break;
2375 }
2376
2298 q = ngx_queue_next(q); 2377 q = ngx_queue_next(q);
2299 2378
2300 f->first = ngx_current_msec; 2379 f->first = ngx_current_msec;
2301 2380
2302 ngx_queue_remove(&f->queue); 2381 ngx_queue_remove(&f->queue);
2303 ngx_queue_insert_tail(&range, &f->queue); 2382 ngx_queue_insert_tail(&range, &f->queue);
2304 2383
2305 len += n; 2384 len += n;
2306 2385
2307 } while (q != ngx_queue_sentinel(&ctx->frames)); 2386 } while (q != ngx_queue_sentinel(&ctx->frames));
2387
2388 if (ngx_queue_empty(&range)) {
2389 break;
2390 }
2308 2391
2309 rc = ngx_quic_send_frames(c, &range); 2392 rc = ngx_quic_send_frames(c, &range);
2310 2393
2311 if (rc == NGX_OK) { 2394 if (rc == NGX_OK) {
2312 /* 2395 /*
2318 ngx_quic_free_frames(c, &range); 2401 ngx_quic_free_frames(c, &range);
2319 2402
2320 } else { 2403 } else {
2321 ngx_queue_add(&ctx->sent, &range); 2404 ngx_queue_add(&ctx->sent, &range);
2322 } 2405 }
2406
2407 cg->in_flight += len;
2408
2409 ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
2410 "quic congestion send if:%uz", cg->in_flight);
2323 2411
2324 } else if (rc == NGX_DONE) { 2412 } else if (rc == NGX_DONE) {
2325 2413
2326 /* no ack is expected for this frames, can free them */ 2414 /* no ack is expected for this frames, can free them */
2327 ngx_quic_free_frames(c, &range); 2415 ngx_quic_free_frames(c, &range);
2384 2472
2385 ctx = ngx_quic_get_send_ctx(c->quic, start->level); 2473 ctx = ngx_quic_get_send_ctx(c->quic, start->level);
2386 2474
2387 ngx_memzero(&pkt, sizeof(ngx_quic_header_t)); 2475 ngx_memzero(&pkt, sizeof(ngx_quic_header_t));
2388 2476
2477 now = ngx_current_msec;
2478
2389 p = src; 2479 p = src;
2390 out.data = src; 2480 out.data = src;
2391 2481
2392 for (q = ngx_queue_head(frames); 2482 for (q = ngx_queue_head(frames);
2393 q != ngx_queue_sentinel(frames); 2483 q != ngx_queue_sentinel(frames);
2407 pkt.need_ack = 1; 2497 pkt.need_ack = 1;
2408 } 2498 }
2409 2499
2410 p += len; 2500 p += len;
2411 f->pnum = ctx->pnum; 2501 f->pnum = ctx->pnum;
2502 f->last = now;
2412 } 2503 }
2413 2504
2414 if (start->level == ssl_encryption_initial) { 2505 if (start->level == ssl_encryption_initial) {
2415 /* ack will not be sent in initial packets due to initial keys being 2506 /* ack will not be sent in initial packets due to initial keys being
2416 * discarded when handshake start. 2507 * discarded when handshake start.
2474 } 2565 }
2475 2566
2476 /* len == NGX_OK || NGX_AGAIN */ 2567 /* len == NGX_OK || NGX_AGAIN */
2477 ctx->pnum++; 2568 ctx->pnum++;
2478 2569
2479 now = ngx_current_msec;
2480 start->last = now;
2481
2482 return pkt.need_ack ? NGX_OK : NGX_DONE; 2570 return pkt.need_ack ? NGX_OK : NGX_DONE;
2483 } 2571 }
2484 2572
2485 2573
2486 static void 2574 static void
2618 2706
2619 ngx_queue_remove(&f->queue); 2707 ngx_queue_remove(&f->queue);
2620 ngx_queue_insert_tail(&range, &f->queue); 2708 ngx_queue_insert_tail(&range, &f->queue);
2621 2709
2622 } while (q != ngx_queue_sentinel(&ctx->sent)); 2710 } while (q != ngx_queue_sentinel(&ctx->sent));
2711
2712 ngx_quic_congestion_lost(c, start->last);
2623 2713
2624 /* NGX_DONE is impossible here, such frames don't get into this queue */ 2714 /* NGX_DONE is impossible here, such frames don't get into this queue */
2625 if (ngx_quic_send_frames(c, &range) != NGX_OK) { 2715 if (ngx_quic_send_frames(c, &range) != NGX_OK) {
2626 return NGX_ERROR; 2716 return NGX_ERROR;
2627 } 2717 }
2779 sn->c->read->log = c->log; 2869 sn->c->read->log = c->log;
2780 sn->c->write->log = c->log; 2870 sn->c->write->log = c->log;
2781 2871
2782 log->connection = sn->c->number; 2872 log->connection = sn->c->number;
2783 2873
2874 if ((id & NGX_QUIC_STREAM_UNIDIRECTIONAL) == 0
2875 || (id & NGX_QUIC_STREAM_SERVER_INITIATED))
2876 {
2877 sn->c->write->ready = 1;
2878 }
2879
2784 cln = ngx_pool_cleanup_add(pool, 0); 2880 cln = ngx_pool_cleanup_add(pool, 0);
2785 if (cln == NULL) { 2881 if (cln == NULL) {
2786 ngx_close_connection(sn->c); 2882 ngx_close_connection(sn->c);
2787 ngx_destroy_pool(pool); 2883 ngx_destroy_pool(pool);
2788 return NULL; 2884 return NULL;
2897 2993
2898 static ssize_t 2994 static ssize_t
2899 ngx_quic_stream_send(ngx_connection_t *c, u_char *buf, size_t size) 2995 ngx_quic_stream_send(ngx_connection_t *c, u_char *buf, size_t size)
2900 { 2996 {
2901 u_char *p, *end; 2997 u_char *p, *end;
2902 size_t fsize, limit; 2998 size_t fsize, limit, n, len;
2999 uint64_t sent, unacked;
2903 ngx_connection_t *pc; 3000 ngx_connection_t *pc;
2904 ngx_quic_frame_t *frame; 3001 ngx_quic_frame_t *frame;
2905 ngx_quic_stream_t *qs; 3002 ngx_quic_stream_t *qs;
2906 ngx_quic_connection_t *qc; 3003 ngx_quic_connection_t *qc;
2907 3004
2921 * 25 = 1 + 8x3 is max header for STREAM frame, with 1 byte for frame type 3018 * 25 = 1 + 8x3 is max header for STREAM frame, with 1 byte for frame type
2922 */ 3019 */
2923 limit = qc->ctp.max_packet_size - NGX_QUIC_MAX_SHORT_HEADER - 25 3020 limit = qc->ctp.max_packet_size - NGX_QUIC_MAX_SHORT_HEADER - 25
2924 - EVP_GCM_TLS_TAG_LEN; 3021 - EVP_GCM_TLS_TAG_LEN;
2925 3022
3023 len = size;
3024 sent = c->sent;
3025 unacked = sent - qs->acked;
3026
3027 if (unacked >= NGX_QUIC_STREAM_BUFSIZE) {
3028 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
3029 "quic send hit buffer size");
3030 len = 0;
3031
3032 } else if (unacked + len > NGX_QUIC_STREAM_BUFSIZE) {
3033 len = NGX_QUIC_STREAM_BUFSIZE - unacked;
3034 }
3035
2926 p = (u_char *) buf; 3036 p = (u_char *) buf;
2927 end = (u_char *) buf + size; 3037 end = (u_char *) buf + len;
3038 n = 0;
2928 3039
2929 while (p < end) { 3040 while (p < end) {
2930 3041
2931 fsize = ngx_min(limit, (size_t) (end - p)); 3042 fsize = ngx_min(limit, (size_t) (end - p));
2932 3043
2949 frame->u.stream.length = fsize; 3060 frame->u.stream.length = fsize;
2950 frame->u.stream.data = frame->data; 3061 frame->u.stream.data = frame->data;
2951 3062
2952 c->sent += fsize; 3063 c->sent += fsize;
2953 p += fsize; 3064 p += fsize;
3065 n += fsize;
2954 3066
2955 ngx_sprintf(frame->info, "stream 0x%xi len=%ui level=%d", 3067 ngx_sprintf(frame->info, "stream 0x%xi len=%ui level=%d",
2956 qs->id, fsize, frame->level); 3068 qs->id, fsize, frame->level);
2957 3069
2958 ngx_quic_queue_frame(qc, frame); 3070 ngx_quic_queue_frame(qc, frame);
2959 } 3071 }
2960 3072
2961 return size; 3073 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
3074 "quic stream send %uz sent:%O, unacked:%uL",
3075 n, c->sent, (uint64_t) c->sent - qs->acked);
3076
3077 if (n != size) {
3078 c->write->ready = 0;
3079 }
3080
3081 if (n == 0) {
3082 return NGX_AGAIN;
3083 }
3084
3085 return n;
2962 } 3086 }
2963 3087
2964 3088
2965 static void 3089 static void
2966 ngx_quic_stream_cleanup_handler(void *data) 3090 ngx_quic_stream_cleanup_handler(void *data)
3120 return frame; 3244 return frame;
3121 } 3245 }
3122 3246
3123 3247
3124 static void 3248 static void
3249 ngx_quic_congestion_ack(ngx_connection_t *c, ngx_quic_frame_t *f)
3250 {
3251 ssize_t n;
3252 ngx_msec_t timer;
3253 ngx_quic_congestion_t *cg;
3254 ngx_quic_connection_t *qc;
3255
3256 qc = c->quic;
3257 cg = &qc->congestion;
3258
3259 n = ngx_quic_create_frame(NULL, f);
3260
3261 cg->in_flight -= n;
3262
3263 timer = f->last - cg->recovery_start;
3264
3265 if ((ngx_msec_int_t) timer <= 0) {
3266 return;
3267 }
3268
3269 if (cg->window < cg->ssthresh) {
3270 cg->window += n;
3271
3272 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
3273 "quic congestion slow start win:%uz, ss:%uz, if:%uz",
3274 cg->window, cg->ssthresh, cg->in_flight);
3275
3276 } else {
3277 cg->window += qc->tp.max_packet_size * n / cg->window;
3278
3279 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
3280 "quic congestion avoidance win:%uz, ss:%uz, if:%uz",
3281 cg->window, cg->ssthresh, cg->in_flight);
3282 }
3283
3284 /* prevent recovery_start from wrapping */
3285
3286 timer = cg->recovery_start - ngx_current_msec + qc->tp.max_idle_timeout * 2;
3287
3288 if ((ngx_msec_int_t) timer < 0) {
3289 cg->recovery_start = ngx_current_msec - qc->tp.max_idle_timeout * 2;
3290 }
3291 }
3292
3293
3294 static void
3295 ngx_quic_congestion_lost(ngx_connection_t *c, ngx_msec_t sent)
3296 {
3297 ngx_msec_t timer;
3298 ngx_quic_congestion_t *cg;
3299 ngx_quic_connection_t *qc;
3300
3301 qc = c->quic;
3302 cg = &qc->congestion;
3303
3304 timer = sent - cg->recovery_start;
3305
3306 if ((ngx_msec_int_t) timer <= 0) {
3307 return;
3308 }
3309
3310 cg->recovery_start = ngx_current_msec;
3311 cg->window /= 2;
3312
3313 if (cg->window < qc->tp.max_packet_size * 2) {
3314 cg->window = qc->tp.max_packet_size * 2;
3315 }
3316
3317 cg->ssthresh = cg->window;
3318
3319 ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
3320 "quic congestion lost win:%uz, ss:%uz, if:%uz",
3321 cg->window, cg->ssthresh, cg->in_flight);
3322 }
3323
3324
3325 static void
3125 ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame) 3326 ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame)
3126 { 3327 {
3127 ngx_quic_connection_t *qc; 3328 ngx_quic_connection_t *qc;
3128 3329
3129 qc = c->quic; 3330 qc = c->quic;