DNS UDP proxy with UNIX socket is not working

[Vladislav Odintsov]

Hi,

things go in such a way, that I need to pass DNS traffic from LXC container to host system without real network between them.
I decided to try NGINX as a proxy server to pass DNS requests/responses via shared unix socket, which is passed from host system as a mountpoint.

I've removed LXC container from my scheme to concentrate on the problem itself, as it reproduces on a normal system without containers involved.

I've got two separate unix sockets: one for tcp-originated requests and one for udp, as nginx configures unix sockets to be stream or dgram based on server's configuration (tcp vs udp).

nginx.conf:

user nginx;
worker_processes 1;
worker_rlimit_nofile 100000;

pid /var/run/nginx.pid;
error_log /var/log/nginx/error.log warn;

events {
    use epoll;
    worker_connections 1024;
    multi_accept on;
}

stream {

    # TCP
    server {
        listen 5353;
        proxy_pass unix:/var/lib/nginx/dns-tcp.sock;
    }

    server {
        listen unix://var/lib/nginx/dns-tcp.sock;
        proxy_pass 10.70.112.1:53;
    }


    # UDP
    server {
        listen 5353 udp;
        proxy_pass unix:/var/lib/nginx/dns-udp.sock;
    }

    server {
        listen unix://var/lib/nginx/dns-udp.sock udp;
        proxy_pass 10.70.112.1:53;
    }
}

For tcp, DNS traffic works excellent:

[root@dev ~]# dig @127.0.0.1 -p 5353 ya.ru +tcp

; <<>> DiG 9.9.4-RedHat-9.9.4-61.el7 <<>> @127.0.0.1 -p 5353 ya.ru +tcp
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 59275
;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
;; QUESTION SECTION:
;ya.ru.				IN	A

;; ANSWER SECTION:
ya.ru.			384	IN	A	87.250.250.242

;; Query time: 2 msec
;; SERVER: 127.0.0.1#5353(127.0.0.1)
;; WHEN: Fri Sep 03 15:20:55 MSK 2021
;; MSG SIZE  rcvd: 50

strace output:

[root@dev ~]# strace -s 1024 -fp 3876008
strace: Process 3876008 attached
epoll_wait(10, [{EPOLLIN, {u32=1176072208, u64=139720757178384}}], 512, 588295) = 1
accept4(5, {sa_family=AF_INET, sin_port=htons(40085), sin_addr=inet_addr("127.0.0.1")}, [16], SOCK_NONBLOCK) = 13
setsockopt(13, SOL_TCP, TCP_NODELAY, [1], 4) = 0
socket(AF_LOCAL, SOCK_STREAM, 0)        = 14
ioctl(14, FIONBIO, [1])                 = 0
epoll_ctl(10, EPOLL_CTL_ADD, 14, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1176073648, u64=139720757179824}}) = 0
connect(14, {sa_family=AF_LOCAL, sun_path="/var/lib/nginx/dns-tcp.sock"}, 110) = 0
epoll_ctl(10, EPOLL_CTL_ADD, 13, {EPOLLIN|EPOLLRDHUP|EPOLLET, {u32=1176073408, u64=139720757179584}}) = 0
accept4(5, 0x7fff487cc150, 0x7fff487cc14c, SOCK_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable)
epoll_wait(10, [{EPOLLOUT, {u32=1176073648, u64=139720757179824}}, {EPOLLIN, {u32=1176072448, u64=139720757178624}}, {EPOLLIN, {u32=1176073408, u64=139720757179584}}], 512, 583915) = 3
accept4(6, {sa_family=AF_LOCAL, NULL}, [2], SOCK_NONBLOCK) = 15
socket(AF_INET, SOCK_STREAM, IPPROTO_IP) = 16
ioctl(16, FIONBIO, [1])                 = 0
epoll_ctl(10, EPOLL_CTL_ADD, 16, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1176074608, u64=139720757180784}}) = 0
connect(16, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.70.112.1")}, 16) = -1 EINPROGRESS (Operation now in progress)
accept4(6, 0x7fff487cc150, 0x7fff487cc14c, SOCK_NONBLOCK) = -1 EAGAIN (Resource temporarily unavailable)
recvfrom(13, "\0\"\347\213\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0", 16384, 0, NULL, NULL) = 36
writev(14, [{"\0\"\347\213\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0", 36}], 1) = 36
epoll_wait(10, [{EPOLLOUT, {u32=1176074608, u64=139720757180784}}], 512, 60000) = 1
getsockopt(16, SOL_SOCKET, SO_ERROR, [0], [4]) = 0
setsockopt(16, SOL_TCP, TCP_NODELAY, [1], 4) = 0
epoll_ctl(10, EPOLL_CTL_ADD, 15, {EPOLLIN|EPOLLRDHUP|EPOLLET, {u32=1176074368, u64=139720757180544}}) = 0
epoll_wait(10, [{EPOLLIN, {u32=1176074368, u64=139720757180544}}], 512, 583913) = 1
recvfrom(15, "\0\"\347\213\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0", 16384, 0, NULL, NULL) = 36
writev(16, [{"\0\"\347\213\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0", 36}], 1) = 36
epoll_wait(10, [{EPOLLOUT, {u32=1176073648, u64=139720757179824}}], 512, 583913) = 1
epoll_wait(10, [{EPOLLIN|EPOLLOUT, {u32=1176074608, u64=139720757180784}}], 512, 583913) = 1
recvfrom(16, "\0002\347\213\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\1\200\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0", 16384, 0, NULL, NULL) = 52
writev(15, [{"\0002\347\213\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\1\200\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0", 52}], 1) = 52
epoll_wait(10, [{EPOLLIN|EPOLLOUT, {u32=1176073648, u64=139720757179824}}], 512, 583912) = 1
recvfrom(14, "\0002\347\213\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\1\200\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0", 16384, 0, NULL, NULL) = 52
writev(13, [{"\0002\347\213\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\1\200\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0", 52}], 1) = 52
epoll_wait(10, [{EPOLLIN|EPOLLRDHUP, {u32=1176073408, u64=139720757179584}}], 512, 583912) = 1
recvfrom(13, "", 16384, 0, NULL, NULL)  = 0
close(14)                               = 0
close(13)                               = 0
epoll_wait(10, [{EPOLLIN|EPOLLHUP|EPOLLRDHUP, {u32=1176074368, u64=139720757180544}}], 512, 583912) = 1
recvfrom(15, "", 16384, 0, NULL, NULL)  = 0
close(16)                               = 0
close(15)                               = 0
epoll_wait(10, ^Cstrace: Process 3876008 detached
 <detached ...>

But in UDP case, nginx process:

1. gets request from dgram unix socket
2. sends request to configured upstream server
3. gets response from configured upstream server
4. tries to send response to unix socket and gets an ECONNREFUSED error and request hangs.

[{EPOLLIN, {u32=1176072688, u64=139720757178864}}], 512, 440326) = 1
recvmsg(7, {msg_name(16)={sa_family=AF_INET, sin_port=htons(55102), sin_addr=inet_addr("127.0.0.1")}, msg_iov(1)=[{"\6\261\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0", 65535}], msg_controllen=32, [{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, {ipi_ifindex=if_nametoindex("lo"), ipi_spec_dst=inet_addr("127.0.0.1"), ipi_addr=inet_addr("127.0.0.1")}}], msg_flags=0}, 0) = 34
socket(AF_LOCAL, SOCK_DGRAM, 0)         = 13
ioctl(13, FIONBIO, [1])                 = 0
epoll_ctl(10, EPOLL_CTL_ADD, 13, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1176074609, u64=139720757180785}}) = 0
connect(13, {sa_family=AF_LOCAL, sun_path="/var/lib/nginx/dns-udp.sock"}, 110) = 0
sendmsg(13, {msg_name(0)=NULL, msg_iov(1)=[{"\6\261\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0", 34}], msg_controllen=0, msg_flags=0}, 0) = 34
recvmsg(7, 0x7fff487cc010, 0)           = -1 EAGAIN (Resource temporarily unavailable)
epoll_wait(10, [{EPOLLOUT, {u32=1176074609, u64=139720757180785}}, {EPOLLIN, {u32=1176072928, u64=139720757179104}}], 512, 438024) = 2
recvmsg(8, {msg_name(0)=0x7fff487cc0a0, msg_iov(1)=[{"\6\261\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0", 65535}], msg_controllen=0, msg_flags=0}, 0) = 34
socket(AF_INET, SOCK_DGRAM, IPPROTO_IP) = 14
ioctl(14, FIONBIO, [1])                 = 0
epoll_ctl(10, EPOLL_CTL_ADD, 14, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=1176073649, u64=139720757179825}}) = 0
connect(14, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.70.112.1")}, 16) = 0
sendmsg(14, {msg_name(0)=NULL, msg_iov(1)=[{"\6\261\1 \0\1\0\0\0\0\0\1\2ya\2ru\0\0\1\0\1\0\0)\20\0\0\0\0\0\0\0", 34}], msg_controllen=0, msg_flags=0}, 0) = 34
recvmsg(8, 0x7fff487cc010, 0)           = -1 EAGAIN (Resource temporarily unavailable)
epoll_wait(10, [{EPOLLOUT, {u32=1176074609, u64=139720757180785}}, {EPOLLOUT, {u32=1176073649, u64=139720757179825}}], 512, 438024) = 2
epoll_wait(10, [{EPOLLIN|EPOLLOUT, {u32=1176073649, u64=139720757179825}}], 512, 438023) = 1
recvfrom(14, "\6\261\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\0\356\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0", 16384, 0, NULL, NULL) = 50
sendmsg(8, {msg_name(16)={sa_family=AF_LOCAL, sun_path=@""}, msg_iov(1)=[{"\6\261\201\200\0\1\0\1\0\0\0\1\2ya\2ru\0\0\1\0\1\300\f\0\1\0\1\0\0\0\356\0\4W\372\372\362\0\0)\20\0\0\0\0\0\0\0", 50}], msg_controllen=0, msg_flags=0}, 0) = -1 ECONNREFUSED (Connection refused)
close(14)                               = 0
epoll_wait(10,

In tcpdump I see request to upstream server and response:

[root@dev ~]# tcpdump  -ni eth0 port 53 and host 10.70.112.1
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
15:24:00.825725 IP 10.70.112.35.55180 > 10.70.112.1.domain: 23283+ [1au] A? ya.ru. (34)
15:24:00.826905 IP 10.70.112.1.domain > 10.70.112.35.55180: 23283 1/0/1 A 87.250.250.242 (50)

Please help understand what could go wrong and how to fix this.
Feel free to ask any additional information.

Thanks.

[Maxim Dounin]

For datagram unix socket to work for server-to-client communication, some temporary receiving socket needs to be created by the client. This is not something nginx currently does, hence the error in the configuration provided.

[Vladislav Odintsov]

Thanks for the feedback.
Should we look for another solution to handle datagram socket or this is something that can be fixed quickly?

I suppose, that either proxy_bind could accept path-to-file as an argument to bind client requests on, or abstract socket can be bound by client?

[Maxim Dounin]

Since each session needs to use an unique path, I don't think there is an easy fix. The most simple (yet probably wrong) approach would be adapt proxy_bind to support unix sockets and so make it possible to explicitly configure something random/unique enough using variables. This needs to be done by someone though.

A readily available solution would be to use AF_INET / AF_INET6 sockets instead, these are working fine.