id	summary	reporter	owner	description	type	status	priority	milestone	component	version	resolution	keywords	cc	uname	nginx_version
2110	nginx as grpc reverse proxy report 499 after few weeks of work	shuliakovsky@…		"NGINX used as grpc-proxy with queue to avoid throttling from grpc service with listed below configuration:

After few thousands of successful proxied requests or after few weeks of good work( Normal work logs)

=============     Normal work logs: ================
10.244.0.10 - - [16/Dec/2020:00:15:36 +0000] ""POST /clarifai.api.V2/PostModelOutputs HTTP/2.0"" 200 1267 ""-"" ""grpc-java-netty/1.29.0""
10.244.0.10 - - [16/Dec/2020:00:15:36 +0000] ""POST /clarifai.api.V2/PostModelOutputs HTTP/2.0"" 200 604 ""-"" ""grpc-java-netty/1.29.0""
10.244.0.10 - - [16/Dec/2020:00:15:36 +0000] ""POST /clarifai.api.V2/PostModelOutputs HTTP/2.0"" 200 1260 ""-"" ""grpc-java-netty/1.29.0""
10.244.0.10 - - [16/Dec/2020:00:15:39 +0000] ""POST /clarifai.api.V2/PostModelOutputs HTTP/2.0"" 200 1514 ""-"" ""grpc-java-netty/1.29.0""
10.244.0.10 - - [16/Dec/2020:00:15:39 +0000] ""POST /clarifai.api.V2/PostModelOutputs HTTP/2.0"" 200 576 ""-"" ""grpc-java-netty/1.29.0""
10.244.0.10 - - [16/Dec/2020:00:15:39 +0000] ""POST /clarifai.api.V2/PostModelOutputs HTTP/2.0"" 200 1258 ""-"" ""grpc-java-netty/1.29.0""
===============================================================

(means randomly) NGINX begins report 499 error on all or on part of workers (Abnormal work logs example). 

=========== Abnormal work logs example ================
10.244.0.88 - - [11/Nov/2020:06:32:22 +0000] ""POST /clarifai.api.V2/PostModelOutputs HTTP/2.0"" 499 0 ""-"" ""grpc-java-netty/1.29.0""
10.244.0.88 - - [11/Nov/2020:06:33:21 +0000] ""POST /clarifai.api.V2/PostModelOutputs HTTP/2.0"" 499 0 ""-"" ""grpc-java-netty/1.29.0""
=======================================================


after ""nginx -s reload"" everything begin works properly for next few weeks or few thousands of requests.


============ begin of nginx.conf ==========
user  nginx;
worker_processes  4;

error_log  /dev/stdout debug;
pid        /var/run/nginx.pid;


events {
  worker_connections  1024;
}

http {
  default_type  application/octet-stream;

  log_format  main  '$remote_addr - $remote_user [$time_local] ""$request"" '
                    '$status $body_bytes_sent ""$http_referer"" ""$limit_req_status""'
                    '""$http_user_agent""';

  access_log  /dev/stdout main;

  sendfile        on;

  keepalive_timeout  2048;

  gzip  on;

  # Limit Rate
  limit_req_zone $server_name zone=perserver:10m rate=8r/s;

  include /etc/nginx/conf.d/*.conf;
}
=============== end of nginx.conf ================

=============== begin of /etc/nginx/conf.d/default.conf ===========
server {
  listen 1080 http2;
  server_name ~^.*$;

  # Logs
  access_log /dev/stdout ;
  error_log /dev/stdout ;

  client_max_body_size 10m;

  # Proxy
  #proxy_ignore_client_abort on;
  proxy_connect_timeout 2048;
  proxy_read_timeout 2048;
  proxy_send_timeout 2048;


  # /Default Location
  location / {
    resolver 1.1.1.1 valid=30s;
    grpc_pass grpc://api-grpc.clarifai.com:18080;
    limit_req zone=perserver burst=10000 delay=1;
    grpc_connect_timeout 240;
    grpc_read_timeout 2048;
    grpc_send_timeout 2048;
    grpc_socket_keepalive on;

  }
  location /clarifai_proxy_status {
    auth_basic ""Restricted Content"";
    auth_basic_user_file /etc/nginx/.htpasswd;
    stub_status;
    access_log off;
  }
}
============ end of /etc/nginx/conf.d/default.conf ==========="	defect	closed	major	nginx-1.19	nginx-module	1.19.x	invalid	grpc module		Linux clarifaiproxy-79bc996599-glr95 4.19.0-0.bpo.6-amd64 #1 SMP Debian 4.19.67-2+deb10u2~bpo9+1 (2019-11-12) x86_64 GNU/Linux	"nginx version: nginx/1.19.5
built by gcc 8.3.0 (Debian 8.3.0-6)
built with OpenSSL 1.1.1d  10 Sep 2019
TLS SNI support enabled
configure arguments: --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-compat --with-file-aio --with-threads --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_mp4_module --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-mail --with-mail_ssl_module --with-stream --with-stream_realip_module --with-stream_ssl_module --with-stream_ssl_preread_module --with-cc-opt='-g -O2 -fdebug-prefix-map=/data/builder/debuild/nginx-1.19.5/debian/debuild-base/nginx-1.19.5=. -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC' --with-ld-opt='-Wl,-z,relro -Wl,-z,now -Wl,--as-needed -pie'"