id summary reporter owner description type status priority milestone component version resolution keywords cc uname nginx_version 2242 DNS UDP proxy with UNIX socket is not working Vladislav Odintsov "Hi, things go in such a way, that I need to pass DNS traffic from LXC container to host system without real network between them. I decided to try NGINX as a proxy server to pass DNS requests/responses via shared unix socket, which is passed from host system as a mountpoint. I've removed LXC container from my scheme to concentrate on the problem itself, as it reproduces on a normal system without containers involved. I've got two separate unix sockets: one for tcp-originated requests and one for udp, as nginx configures unix sockets to be stream or dgram based on server's configuration (tcp vs udp). nginx.conf: {{{ user nginx; worker_processes 1; worker_rlimit_nofile 100000; pid /var/run/nginx.pid; error_log /var/log/nginx/error.log warn; events { use epoll; worker_connections 1024; multi_accept on; } stream { # TCP server { listen 5353; proxy_pass unix:/var/lib/nginx/dns-tcp.sock; } server { listen unix://var/lib/nginx/dns-tcp.sock; proxy_pass 10.70.112.1:53; } # UDP server { listen 5353 udp; proxy_pass unix:/var/lib/nginx/dns-udp.sock; } server { listen unix://var/lib/nginx/dns-udp.sock udp; proxy_pass 10.70.112.1:53; } } }}} For tcp, DNS traffic works excellent: {{{ [root@dev ~]# dig @127.0.0.1 -p 5353 ya.ru +tcp ; <<>> DiG 9.9.4-RedHat-9.9.4-61.el7 <<>> @127.0.0.1 -p 5353 ya.ru +tcp ; (1 server found) ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 59275 ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 4096 ;; QUESTION SECTION: ;ya.ru. IN A ;; ANSWER SECTION: ya.ru. 384 IN A 87.250.250.242 ;; Query time: 2 msec ;; SERVER: 127.0.0.1#5353(127.0.0.1) ;; WHEN: Fri Sep 03 15:20:55 MSK 2021 ;; MSG SIZE rcvd: 50 }}} strace output: {{{ [root@dev ~]# strace -s 1024 -fp 3876008 strace: Process 3876008 attached epoll_wait(10, [{EPOLLIN, {u32=1176072208, u64=139720757178384}}], 512, 588295) = 1 accept4(5, {sa_family=AF_INET, sin_port=htons(40085), sin_addr=inet_addr(""127.0.0.1"")}, [16], SOCK_NONBLOCK) = 13 setsockopt(13, SOL_TCP, TCP_NODELAY, [1], 4) = 0 socket(AF_LOCAL, SOCK_STREAM, 0) = 14 ioctl(14, FIONBIO, [1]) = 0 epoll_ctl(10, EPOLL_CTL_ADD, 14, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1176073648, u64=139720757179824}}) = 0 connect(14, {sa_family=AF_LOCAL, sun_path=""/var/lib/nginx/dns-tcp.sock""}, 110) = 0 epoll_ctl(10, EPOLL_CTL_ADD, 13, {EPOLLIN|EPOLLRDHUP|EPOLLET, {u32=1176073408, u64=139720757179584}}) = 0 accept4(5, 0x7fff487cc150, 0x7fff487cc14c, SOCK_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable) epoll_wait(10, [{EPOLLOUT, {u32=1176073648, u64=139720757179824}}, {EPOLLIN, {u32=1176072448, u64=139720757178624}}, {EPOLLIN, {u32=1176073408, u64=139720757179584}}], 512, 583915) = 3 accept4(6, {sa_family=AF_LOCAL, NULL}, [2], SOCK_NONBLOCK) = 15 socket(AF_INET, SOCK_STREAM, IPPROTO_IP) = 16 ioctl(16, FIONBIO, [1]) = 0 epoll_ctl(10, EPOLL_CTL_ADD, 16, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1176074608, u64=139720757180784}}) = 0 connect(16, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr(""10.70.112.1"")}, 16) = -1 EINPROGRESS (Operation now in progress) accept4(6, 0x7fff487cc150, 0x7fff487cc14c, SOCK_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable) recvfrom(13, ""\0\""\347\213\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0"", 16384, 0, NULL, NULL) = 36 writev(14, [{""\0\""\347\213\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0"", 36}], 1) = 36 epoll_wait(10, [{EPOLLOUT, {u32=1176074608, u64=139720757180784}}], 512, 60000) = 1 getsockopt(16, SOL_SOCKET, SO_ERROR, [0], [4]) = 0 setsockopt(16, SOL_TCP, TCP_NODELAY, [1], 4) = 0 epoll_ctl(10, EPOLL_CTL_ADD, 15, {EPOLLIN|EPOLLRDHUP|EPOLLET, {u32=1176074368, u64=139720757180544}}) = 0 epoll_wait(10, [{EPOLLIN, {u32=1176074368, u64=139720757180544}}], 512, 583913) = 1 recvfrom(15, ""\0\""\347\213\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0"", 16384, 0, NULL, NULL) = 36 writev(16, [{""\0\""\347\213\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0"", 36}], 1) = 36 epoll_wait(10, [{EPOLLOUT, {u32=1176073648, u64=139720757179824}}], 512, 583913) = 1 epoll_wait(10, [{EPOLLIN|EPOLLOUT, {u32=1176074608, u64=139720757180784}}], 512, 583913) = 1 recvfrom(16, ""\0002\347\213\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\1\200\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0"", 16384, 0, NULL, NULL) = 52 writev(15, [{""\0002\347\213\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\1\200\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0"", 52}], 1) = 52 epoll_wait(10, [{EPOLLIN|EPOLLOUT, {u32=1176073648, u64=139720757179824}}], 512, 583912) = 1 recvfrom(14, ""\0002\347\213\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\1\200\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0"", 16384, 0, NULL, NULL) = 52 writev(13, [{""\0002\347\213\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\1\200\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0"", 52}], 1) = 52 epoll_wait(10, [{EPOLLIN|EPOLLRDHUP, {u32=1176073408, u64=139720757179584}}], 512, 583912) = 1 recvfrom(13, """", 16384, 0, NULL, NULL) = 0 close(14) = 0 close(13) = 0 epoll_wait(10, [{EPOLLIN|EPOLLHUP|EPOLLRDHUP, {u32=1176074368, u64=139720757180544}}], 512, 583912) = 1 recvfrom(15, """", 16384, 0, NULL, NULL) = 0 close(16) = 0 close(15) = 0 epoll_wait(10, ^Cstrace: Process 3876008 detached }}} But in UDP case, nginx process: 1. gets request from dgram unix socket 2. sends request to configured upstream server 3. gets response from configured upstream server 4. tries to send response to unix socket and gets an ECONNREFUSED error and request hangs. {{{ [{EPOLLIN, {u32=1176072688, u64=139720757178864}}], 512, 440326) = 1 recvmsg(7, {msg_name(16)={sa_family=AF_INET, sin_port=htons(55102), sin_addr=inet_addr(""127.0.0.1"")}, msg_iov(1)=[{""\6\261\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0"", 65535}], msg_controllen=32, [{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, {ipi_ifindex=if_nametoindex(""lo""), ipi_spec_dst=inet_addr(""127.0.0.1""), ipi_addr=inet_addr(""127.0.0.1"")}}], msg_flags=0}, 0) = 34 socket(AF_LOCAL, SOCK_DGRAM, 0) = 13 ioctl(13, FIONBIO, [1]) = 0 epoll_ctl(10, EPOLL_CTL_ADD, 13, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1176074609, u64=139720757180785}}) = 0 connect(13, {sa_family=AF_LOCAL, sun_path=""/var/lib/nginx/dns-udp.sock""}, 110) = 0 sendmsg(13, {msg_name(0)=NULL, msg_iov(1)=[{""\6\261\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0"", 34}], msg_controllen=0, msg_flags=0}, 0) = 34 recvmsg(7, 0x7fff487cc010, 0) = -1 EAGAIN (Resource temporarily unavailable) epoll_wait(10, [{EPOLLOUT, {u32=1176074609, u64=139720757180785}}, {EPOLLIN, {u32=1176072928, u64=139720757179104}}], 512, 438024) = 2 recvmsg(8, {msg_name(0)=0x7fff487cc0a0, msg_iov(1)=[{""\6\261\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0"", 65535}], msg_controllen=0, msg_flags=0}, 0) = 34 socket(AF_INET, SOCK_DGRAM, IPPROTO_IP) = 14 ioctl(14, FIONBIO, [1]) = 0 epoll_ctl(10, EPOLL_CTL_ADD, 14, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1176073649, u64=139720757179825}}) = 0 connect(14, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr(""10.70.112.1"")}, 16) = 0 sendmsg(14, {msg_name(0)=NULL, msg_iov(1)=[{""\6\261\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0"", 34}], msg_controllen=0, msg_flags=0}, 0) = 34 recvmsg(8, 0x7fff487cc010, 0) = -1 EAGAIN (Resource temporarily unavailable) epoll_wait(10, [{EPOLLOUT, {u32=1176074609, u64=139720757180785}}, {EPOLLOUT, {u32=1176073649, u64=139720757179825}}], 512, 438024) = 2 epoll_wait(10, [{EPOLLIN|EPOLLOUT, {u32=1176073649, u64=139720757179825}}], 512, 438023) = 1 recvfrom(14, ""\6\261\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\0\356\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0"", 16384, 0, NULL, NULL) = 50 sendmsg(8, {msg_name(16)={sa_family=AF_LOCAL, sun_path=@""""}, msg_iov(1)=[{""\6\261\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\0\356\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0"", 50}], msg_controllen=0, msg_flags=0}, 0) = -1 ECONNREFUSED (Connection refused) close(14) = 0 epoll_wait(10, }}} In tcpdump I see request to upstream server and response: {{{ [root@dev ~]# tcpdump -ni eth0 port 53 and host 10.70.112.1 tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 15:24:00.825725 IP 10.70.112.35.55180 > 10.70.112.1.domain: 23283+ [1au] A? ya.ru. (34) 15:24:00.826905 IP 10.70.112.1.domain > 10.70.112.35.55180: 23283 1/0/1 A 87.250.250.242 (50) }}} Please help understand what could go wrong and how to fix this. Feel free to ask any additional information. Thanks." defect accepted minor nginx-core unix socket, stream module, dns proxy Linux dev 3.10.0-862.34.1.el7.x86_64 #1 SMP Fri May 24 13:20:25 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux "nginx version: nginx/1.21.2 built by gcc 4.8.5 20150623 (Red Hat 4.8.5-28) (GCC) built with OpenSSL 1.0.2k-fips 26 Jan 2017 TLS SNI support enabled configure arguments: --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib64/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-compat --with-file-aio --with-threads --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_mp4_module --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-mail --with-mail_ssl_module --with-stream --with-stream_realip_module --with-stream_ssl_module --with-stream_ssl_preread_module --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -fPIC' --with-ld-opt='-Wl,-z,relro -Wl,-z,now -pie'"