Core dump when http2 client cancelled stream while upstream

[crasyangel.lhy@…]

It would cause segment fault when test as below

---

test cmd

while(true) do curl -k -svo /dev/null "https://www.baidu.com:7023/xxx" --resolve www.baidu.com:7023:127.0.0.1 -T "somefile"& sleep 1; killall curl; done

---

bt stack

#0  ngx_http_upstream_handler (ev=0x7f8122788190) at src/http/ngx_http_upstream.c:1275
1275	    ngx_http_set_log_request(c->log, r);
warning: File "/home/work/jorcol/.gdbinit" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load".
(gdb) bt
#0  ngx_http_upstream_handler (ev=0x7f8122788190) at src/http/ngx_http_upstream.c:1275
#1  0x00000000004300fd in ngx_event_expire_timers () at src/event/ngx_event_timer.c:94
#2  0x000000000042fd6a in ngx_process_events_and_timers (cycle=cycle@entry=0x2054650) at src/event/ngx_event.c:256
#3  0x0000000000436c74 in ngx_worker_process_cycle (cycle=0x2054650, data=<optimized out>) at src/os/unix/ngx_process_cycle.c:750
#4  0x00000000004353a2 in ngx_spawn_process (cycle=cycle@entry=0x2054650, proc=0x436c03 <ngx_worker_process_cycle>, data=0x0,
    name=0x4a1537 "worker process", respawn=respawn@entry=0) at src/os/unix/ngx_process.c:199
#5  0x00000000004375ee in ngx_reap_children (cycle=0x2054650) at src/os/unix/ngx_process_cycle.c:622
#6  ngx_master_process_cycle (cycle=cycle@entry=0x2054650) at src/os/unix/ngx_process_cycle.c:175
#7  0x00000000004125d5 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:382

---

nginx.conf

worker_processes 1;
error_log logs/error.log debug;
events {
    use epoll;
    worker_connections 32768;
}
http {
    access_log logs/access.log.pipe;

    proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
    proxy_connect_timeout 10;
    proxy_read_timeout 10;
    proxy_intercept_errors on;
    proxy_max_temp_file_size 0;
    proxy_buffers 32 16k;
    proxy_busy_buffers_size 64k;
    proxy_request_buffering off;

        server {
            listen 7023 ssl http2;
            ssl_certificate      server.crt;
            ssl_certificate_key  server.key;
            ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
            ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS:!DES';
            location / {
                proxy_pass http://127.0.0.1:18080;
                error_page 502 504 = @pass;
            }
            location @pass {
                proxy_pass http://127.0.0.1:8000;
            }
}
        server {
            listen 18080;
            location / {
                return 502;
            }
        }
        server {
            listen 8000;
            location / {
                return 200 "haha";
            }
        }
}

---

The request which r=0x2093050, call ngx_http_upstream_init 3 times, and result in coredump finally

#0  ngx_http_upstream_init (r=0x2093050) at src/http/ngx_http_upstream.c:514
#1  0x00000000004535c1 in ngx_http_read_client_request_body (r=r@entry=0x2093050, post_handler=0x45f4f3 <ngx_http_upstream_init>) at src/http/ngx_http_request_body.c:201
#2  0x0000000000489598 in ngx_http_proxy_handler (r=0x2093050) at src/http/modules/ngx_http_proxy_module.c:937
#3  0x0000000000447c9b in ngx_http_core_content_phase (r=0x2093050, ph=<optimized out>) at src/http/ngx_http_core_module.c:1169
#4  0x0000000000442e73 in ngx_http_core_run_phases (r=r@entry=0x2093050) at src/http/ngx_http_core_module.c:858
#5  0x0000000000442f82 in ngx_http_handler (r=r@entry=0x2093050) at src/http/ngx_http_core_module.c:841
#6  0x000000000044ccea in ngx_http_process_request (r=r@entry=0x2093050) at src/http/ngx_http_request.c:2055
#7  0x000000000047726b in ngx_http_v2_run_request (r=0x2093050) at src/http/v2/ngx_http_v2.c:3789
#8  0x00000000004772f7 in ngx_http_v2_state_header_complete (h2c=h2c@entry=0x20530c0, pos=pos@entry=0x7f812487604a "adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\nbce"..., end=end@entry=0x7f812487604a "adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\nbce"...) at src/http/v2/ngx_http_v2.c:1704
#9  0x0000000000477b9e in ngx_http_v2_state_process_header (h2c=h2c@entry=0x20530c0, pos=pos@entry=0x7f812487604a "adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\nbce"..., end=end@entry=0x7f812487604a "adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\nbce"...) at src/http/v2/ngx_http_v2.c:1675
#10 0x0000000000477d94 in ngx_http_v2_state_field_huff (h2c=h2c@entry=0x20530c0, pos=0x7f812487604a "adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\nbce"..., end=end@entry=0x7f812487604a "adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\nbce"...) at src/http/v2/ngx_http_v2.c:1419
#11 0x0000000000478168 in ngx_http_v2_state_field_len (h2c=h2c@entry=0x20530c0, pos=0x7f8124876047 "lA\017adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\n"..., end=end@entry=0x7f812487604a "adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\nbce"...) at src/http/v2/ngx_http_v2.c:1377
#12 0x0000000000478389 in ngx_http_v2_state_header_block (h2c=0x20530c0, pos=<optimized out>, end=0x7f812487604a "adingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostatic\nbase64\nbav\nbav_updown\nbbsliulanqi\nbce"...) at src/http/v2/ngx_http_v2.c:1293
#13 0x000000000047628f in ngx_http_v2_read_handler (rev=rev@entry=0x7f8122788130) at src/http/v2/ngx_http_v2.c:413
#14 0x00000000004767ac in ngx_http_v2_idle_handler (rev=0x7f8122788130) at src/http/v2/ngx_http_v2.c:4536
#15 0x0000000000438897 in ngx_epoll_process_events (cycle=<optimized out>, timer=<optimized out>, flags=<optimized out>) at src/event/modules/ngx_epoll_module.c:902
#16 0x000000000042fd38 in ngx_process_events_and_timers (cycle=cycle@entry=0x2054650) at src/event/ngx_event.c:242
#17 0x0000000000436c74 in ngx_worker_process_cycle (cycle=0x2054650, data=<optimized out>) at src/os/unix/ngx_process_cycle.c:750
#18 0x00000000004353a2 in ngx_spawn_process (cycle=cycle@entry=0x2054650, proc=0x436c03 <ngx_worker_process_cycle>, data=0x0, name=0x4a1537 "worker process", respawn=respawn@entry=0) at src/os/unix/ngx_process.c:199
#19 0x00000000004375ee in ngx_reap_children (cycle=0x2054650) at src/os/unix/ngx_process_cycle.c:622
#20 ngx_master_process_cycle (cycle=cycle@entry=0x2054650) at src/os/unix/ngx_process_cycle.c:175
#21 0x00000000004125d5 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:382

#0  ngx_http_upstream_init (r=0x2093050) at src/http/ngx_http_upstream.c:514
#1  0x000000000045331a in ngx_http_read_client_request_body (r=r@entry=0x2093050, post_handler=0x45f4f3 <ngx_http_upstream_init>) at src/http/ngx_http_request_body.c:45
#2  0x0000000000489598 in ngx_http_proxy_handler (r=0x2093050) at src/http/modules/ngx_http_proxy_module.c:937
#3  0x0000000000447c9b in ngx_http_core_content_phase (r=0x2093050, ph=<optimized out>) at src/http/ngx_http_core_module.c:1169
#4  0x0000000000442e73 in ngx_http_core_run_phases (r=r@entry=0x2093050) at src/http/ngx_http_core_module.c:858
#5  0x0000000000448dc6 in ngx_http_named_location (r=r@entry=0x2093050, name=name@entry=0x7ffc5e6c03c0) at src/http/ngx_http_core_module.c:2520
#6  0x000000000044970b in ngx_http_send_error_page (err_page=0x206d778, r=0x2093050) at src/http/ngx_http_special_response.c:623
#7  ngx_http_special_response_handler (r=r@entry=0x2093050, error=error@entry=502) at src/http/ngx_http_special_response.c:466
#8  0x000000000044bfd7 in ngx_http_finalize_request (r=r@entry=0x2093050, rc=rc@entry=502) at src/http/ngx_http_request.c:2481
#9  0x000000000045a5ac in ngx_http_upstream_finalize_request (r=r@entry=0x2093050, u=u@entry=0x20947f0, rc=rc@entry=502) at src/http/ngx_http_upstream.c:4429
#10 0x000000000045c6de in ngx_http_upstream_intercept_errors (u=0x20947f0, r=0x2093050) at src/http/ngx_http_upstream.c:2653
#11 ngx_http_upstream_process_header (r=0x2093050, u=0x20947f0) at src/http/ngx_http_upstream.c:2430
#12 0x000000000045a775 in ngx_http_upstream_handler (ev=<optimized out>) at src/http/ngx_http_upstream.c:1289
#13 0x0000000000438897 in ngx_epoll_process_events (cycle=<optimized out>, timer=<optimized out>, flags=<optimized out>) at src/event/modules/ngx_epoll_module.c:902
#14 0x000000000042fd38 in ngx_process_events_and_timers (cycle=cycle@entry=0x2054650) at src/event/ngx_event.c:242
#15 0x0000000000436c74 in ngx_worker_process_cycle (cycle=0x2054650, data=<optimized out>) at src/os/unix/ngx_process_cycle.c:750
#16 0x00000000004353a2 in ngx_spawn_process (cycle=cycle@entry=0x2054650, proc=0x436c03 <ngx_worker_process_cycle>, data=0x0, name=0x4a1537 "worker process", respawn=respawn@entry=0) at src/os/unix/ngx_process.c:199
#17 0x00000000004375ee in ngx_reap_children (cycle=0x2054650) at src/os/unix/ngx_process_cycle.c:622
#18 ngx_master_process_cycle (cycle=cycle@entry=0x2054650) at src/os/unix/ngx_process_cycle.c:175
#19 0x00000000004125d5 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:382

#0  ngx_http_upstream_init (r=0x2093050) at src/http/ngx_http_upstream.c:514
#1  0x00000000004741f2 in ngx_http_v2_process_request_body (r=r@entry=0x2093050, pos=pos@entry=0x7f8124876022 "91_app_download\n91_download\nairticket\naladingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostat"..., size=size@entry=5211, last=1) at src/http/v2/ngx_http_v2.c:4010
#2  0x00000000004759bb in ngx_http_v2_state_read_data (h2c=h2c@entry=0x20530c0, pos=pos@entry=0x7f8124876022 "91_app_download\n91_download\nairticket\naladingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostat"..., end=end@entry=0x7f812487747d "oshuo\nyingxiao\nyishengdapp\nyouxi\nyouxiadstatic\nyouxistatic\nyun\nzhanzhangstatic\nzhidahaostatic\nzhidao\nziyouziyuan\nzongheng\n") at src/http/v2/ngx_http_v2.c:959
#3  0x0000000000475df6 in ngx_http_v2_state_data (h2c=0x20530c0, pos=0x7f8124876022 "91_app_download\n91_download\nairticket\naladingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostat"..., end=0x7f812487747d "oshuo\nyingxiao\nyishengdapp\nyouxi\nyouxiadstatic\nyouxistatic\nyun\nzhanzhangstatic\nzhidahaostatic\nzhidao\nziyouziyuan\nzongheng\n") at src/http/v2/ngx_http_v2.c:921
#4  0x0000000000474df7 in ngx_http_v2_state_head (h2c=0x20530c0, pos=0x7f8124876022 "91_app_download\n91_download\nairticket\naladingapp\nalcfeedback\napkdl\napmstatic\nappres2_bos_moborobo\nappsearch\nappstore\napptuan\nautoinstall\nbae\nbaifubaostatic\nbaijiastatic\nbaike\nbaikebcs\nbaobaozhidaostat"..., end=0x7f812487747d "oshuo\nyingxiao\nyishengdapp\nyouxi\nyouxiadstatic\nyouxistatic\nyun\nzhanzhangstatic\nzhidahaostatic\nzhidao\nziyouziyuan\nzongheng\n") at src/http/v2/ngx_http_v2.c:788
#5  0x000000000047628f in ngx_http_v2_read_handler (rev=0x7f8122788130) at src/http/v2/ngx_http_v2.c:413
#6  0x0000000000438897 in ngx_epoll_process_events (cycle=<optimized out>, timer=<optimized out>, flags=<optimized out>) at src/event/modules/ngx_epoll_module.c:902
#7  0x000000000042fd38 in ngx_process_events_and_timers (cycle=cycle@entry=0x2054650) at src/event/ngx_event.c:242
#8  0x0000000000436c74 in ngx_worker_process_cycle (cycle=0x2054650, data=<optimized out>) at src/os/unix/ngx_process_cycle.c:750
#9  0x00000000004353a2 in ngx_spawn_process (cycle=cycle@entry=0x2054650, proc=0x436c03 <ngx_worker_process_cycle>, data=0x0, name=0x4a1537 "worker process", respawn=respawn@entry=0) at src/os/unix/ngx_process.c:199
#10 0x00000000004375ee in ngx_reap_children (cycle=0x2054650) at src/os/unix/ngx_process_cycle.c:622
#11 ngx_master_process_cycle (cycle=cycle@entry=0x2054650) at src/os/unix/ngx_process_cycle.c:175
#12 0x00000000004125d5 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:382

Reporter:
GaoYan, Baidu

[Maxim Dounin]

Note that proxying to a different server won't work due to proxy_request_buffering off; in your configuration. While it certainly shouldn't segfault, the configuration in question is not expected to work.

[Sergey Kandaurov <pluknet@…>]

In 7561:9f1f9d6e056a/nginx:

HTTP/2: discard remaining request body after redirect.

Previously, if unbuffered request body reading wasn't finished before
the request was redirected to a different location using error_page
or X-Accel-Redirect, and the request body is read again, this could
lead to disastrous effects, such as a duplicate post_handler call or
"http request count is zero" alert followed by a segmentation fault.

This happened in the following configuration (ticket #1819):

location / {

proxy_request_buffering off;
proxy_pass ​http://bad;
proxy_intercept_errors on;
error_page 502 = /error;

}

location /error {

proxy_pass ​http://backend;

}

[Sergey Kandaurov <pluknet@…>]

In 1498:63a74974a0e9/nginx-tests:

Tests: HTTP/2 unbuffered request body with redirect (ticket #1819).

[Sergey Kandaurov]

A bandaid was added to prevent segfault.