openresty resolver issue
bsiara opened this issue · comments
Time to time my openresty is failing resolve cloudfront name and throw error:
2023/10/20 01:46:51 [error] 528660#528660: *131910618 upstream timed out (110: Operation timed out) while connecting to upstream, client: 31.61.245.62, server: server.name, request: "GET /login.html HTTP/2.0", upstream: "https://143.204.101.10:443/login.html", host: "server.name"
2023/10/20 01:47:51 [error] 528660#528660: *131910618 upstream timed out (110: Operation timed out) while connecting to upstream, client: 31.61.245.62, server: server.name, request: "GET /login.html HTTP/2.0", upstream: "https://143.204.101.99:443/login.html", host: "server.name"
2023/10/20 01:48:51 [error] 528660#528660: *131910618 upstream timed out (110: Operation timed out) while connecting to upstream, client: 31.61.245.62, server: server.name, request: "GET /login.html HTTP/2.0", upstream: "https://143.204.101.195:443/login.html", host: "server.name"
the problem persists until I reload or restart the nginx server
openresty version:
openresty -V
nginx version: openresty/1.21.4.2
built by gcc 12.2.1 20220924 (Alpine 12.2.1_git20220924-r10)
built with OpenSSL 3.1.2 1 Aug 2023
TLS SNI support enabled
configure arguments: --prefix=/opt/nginx/nginx --with-debug --with-cc-opt='-DNGX_LUA_USE_ASSERT -DNGX_LUA_ABORT_AT_PANIC -O2 -fstack-protector-strong' --add-module=../ngx_devel_kit-0.3.2 --add-module=../echo-nginx-module-0.63 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2 --add-module=../set-misc-nginx-module-0.33 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.09 --add-module=../srcache-nginx-module-0.33 --add-module=../ngx_lua-0.10.25 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.34 --add-module=../array-var-nginx-module-0.06 --add-module=../memc-nginx-module-0.19 --add-module=../redis2-nginx-module-0.15 --add-module=../redis-nginx-module-0.3.9 --add-module=../rds-json-nginx-module-0.16 --add-module=../rds-csv-nginx-module-0.09 --add-module=../ngx_stream_lua-0.0.13 --with-ld-opt=-Wl,-rpath,/opt/nginx/luajit/lib --sbin-path=/opt/nginx/sbin/nginx --modules-path=/opt/nginx/etc/nginx/modules --conf-path=/opt/nginx/etc/nginx/nginx.conf --error-log-path=/opt/nginx/logs/error.log --pid-path=/opt/nginx/run/nginx.pid --lock-path=/opt/nginx/run/nginx.lock --user=nginx --group=nginx --with-select_module --with-poll_module --with-threads --with-file-aio --with-http_ssl_module --with-http_v2_module --with-http_realip_module --with-http_addition_module --with-http_xslt_module=dynamic --with-http_image_filter_module=dynamic --with-http_geoip_module=dynamic --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_auth_request_module --with-http_random_index_module --with-http_secure_link_module --with-http_degradation_module --with-http_slice_module --with-http_stub_status_module --with-http_perl_module=dynamic --http-log-path=/opt/nginx/logs/access.log --http-client-body-temp-path=/opt/nginx/cache/client_temp --http-proxy-temp-path=/opt/nginx/cache/proxy_temp --http-fastcgi-temp-path=/opt/nginx/cache/fastcgi_temp --http-uwsgi-temp-path=/opt/nginx/cache/uwsgi_temp --http-scgi-temp-path=/opt/nginx/cache/scgi_temp --with-perl_modules_path=/usr/lib/perl5/vendor_perl --with-mail=dynamic --with-mail_ssl_module --with-stream=dynamic --with-stream_ssl_module --with-stream_realip_module --with-stream_geoip_module=dynamic --with-stream_ssl_preread_module --with-pcre-jit --add-module=/opt/nginx/openresty-1.21.4.2/../naxsi/naxsi_src --add-module=/opt/nginx/openresty-1.21.4.2/../nginx-module-vts --add-module=/opt/nginx/openresty-1.21.4.2/../ngx_brotli --add-dynamic-module=/opt/nginx/openresty-1.21.4.2/../nginx-opentracing/opentracing --with-compat --with-stream
nginx.conf:
worker_processes 2;
worker_cpu_affinity 10 01;
worker_rlimit_nofile 100000;
include /opt/nginx/etc/nginx/modules/*.conf;
pcre_jit on;
events {
worker_connections 20480;
use epoll;
multi_accept on;
}
http {
open_file_cache max=200000 inactive=20s;
open_file_cache_valid 30s;
open_file_cache_min_uses 2;
open_file_cache_errors on;
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_iso8601] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$request_time" "$upstream_connect_time" "$upstream_response_time" '
"$http_host"';
log_format loggerjson escape=json '{"nginx": {"access": {"remote_ip": "$remote_addr", "user_name": "$remote_user", "time": "$msec", "request": "$request", "response_code": $status, "body_sent": { "bytes": $body_bytes_sent }, "referrer": "$http_referer", "agent": "$http_user_agent", "request_time": "$request_time", "upstream_connect_time": "$upstream_connect_time", "upstream_response_time": "$upstream_response_time", "http_host": "$http_host"}}}';
include /opt/nginx/etc/nginx/modsec/waf_rules/*.rules;
include /opt/nginx/naxsi/naxsi_config/naxsi_core.rules;
include /opt/nginx/naxsi-rules/*.rules;
vhost_traffic_status_zone shared:vhost_traffic_status:32m;
access_log /opt/nginx/logs/access.log loggerjson;
error_log /opt/nginx/logs/error.log;
lua_package_path "/opt/nginx/lualib/resty/?.lua;/opt/nginx/site/lualib/?.lua;/opt/nginx/lua/?.lua;;";
lua_shared_dict discovery 1m;
lua_shared_dict jwks 1m;
lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
sendfile on;
tcp_nopush on;
keepalive_timeout 35;
types_hash_max_size 2048;
server_tokens off;
resolver 192.168.100.2 ipv6=off; # point to aws dns
resolver_timeout 10s;
client_body_buffer_size 32M;
client_header_buffer_size 16k;
large_client_header_buffers 8 16k;
client_max_body_size 64M;
proxy_buffer_size 32k;
proxy_buffers 8 16k;
server_names_hash_max_size 1024;
server_names_hash_bucket_size 128;
proxy_headers_hash_max_size 1024;
proxy_headers_hash_bucket_size 128;
variables_hash_max_size 2048;
variables_hash_bucket_size 128;
gzip on;
gzip_min_length 10240;
gzip_comp_level 1;
gzip_vary on;
gzip_disable msie6;
gzip_proxied expired no-cache no-store private auth;
gzip_types
text/css
text/javascript
text/xml
text/plain
text/x-component
application/javascript
application/x-javascript
application/json
application/xml
application/rss+xml
application/atom+xml
font/truetype
font/opentype
application/vnd.ms-fontobject
image/svg+xml;
reset_timedout_connection on;
ssl_session_cache shared:SSL:10m;
server {
listen 80 default_server reuseport backlog=8192;
server_name _;
location / {
return 302 https://$host$request_uri;
}
}
server {
listen 443 ssl http2 default_server reuseport backlog=8192;
ssl_certificate /opt/nginx/etc/nginx/certs/localhost.crt;
ssl_certificate_key /opt/nginx/etc/nginx/certs/localhost.key;
include /opt/nginx/etc/nginx/ssl_params.conf;
error_page 500 502 503 504 /50x.html;
location = /50x.html {
}
}
include /opt/nginx/etc/nginx/conf.d/*.conf;
}
vhost_to_aws_cloudfront.conf
upstream upawscf {
least_conn;
server name.cloudfront.net:443 max_fails=0;
}
server {
client_max_body_size 32M;
listen 443 ssl http2;
server_name server.name;
ssl_certificate /opt/nginx/etc/nginx/certs/server.name.crt;
ssl_certificate_key /opt/nginx/etc/nginx/certs/server.name.key;
include /opt/nginx/etc/nginx/ssl_params.conf;
access_log /opt/nginx/logs/server.name.access.log loggerjson;
location / {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_read_timeout 600s;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Original-URI $request_uri;
proxy_pass_request_headers on;
proxy_pass_request_body on;
proxy_ssl_session_reuse on;
proxy_set_header Host name.cloudfront.net;
proxy_ssl_name name.cloudfront.net;
proxy_ssl_server_name on;
proxy_pass https://upawscf;
}
}
openresty is running in docker
docker -v
Docker version 24.0.1, build 6802122
uname -a
Linux aws-openresty 5.15.0-1036-aws #40~20.04.1-Ubuntu SMP Mon Apr 24 00:21:13 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
systemd-resolve --status
Global
LLMNR setting: no
MulticastDNS setting: no
DNSOverTLS setting: no
DNSSEC setting: no
DNSSEC supported: no
DNSSEC NTA: 10.in-addr.arpa
16.172.in-addr.arpa
168.192.in-addr.arpa
17.172.in-addr.arpa
18.172.in-addr.arpa
19.172.in-addr.arpa
20.172.in-addr.arpa
21.172.in-addr.arpa
22.172.in-addr.arpa
23.172.in-addr.arpa
24.172.in-addr.arpa
25.172.in-addr.arpa
26.172.in-addr.arpa
27.172.in-addr.arpa
28.172.in-addr.arpa
29.172.in-addr.arpa
30.172.in-addr.arpa
31.172.in-addr.arpa
corp
d.f.ip6.arpa
home
internal
intranet
lan
local
private
test
Link 3 (docker0)
Current Scopes: none
DefaultRoute setting: no
LLMNR setting: yes
MulticastDNS setting: no
DNSOverTLS setting: no
DNSSEC setting: no
DNSSEC supported: no
Link 2 (ens5)
Current Scopes: DNS
DefaultRoute setting: yes
LLMNR setting: yes
MulticastDNS setting: no
DNSOverTLS setting: no
DNSSEC setting: no
DNSSEC supported: no
Current DNS Server: 192.168.100.2
DNS Servers: 192.168.100.2
DNS Domain: domain.name
At the same time when openresty throw errorI had a script running with curl corectly resolve cloudfront name and get proper ip and response code
script.sh
#!/bin/bash
elog () {
echo `date '+%Y-%m-%d %H:%M:%S'`" $1"
}
main () {
while true; do
elog "Start server.name checking" >> script.log
print_header >> script.log
make_request_servername
elog "End server.name checking" >> script.log
sleep 1m
done
}
print_header () {
echo "code;time_total;time_connect;time_appconnect;time_starttransfer;time_namelookup"
}
make_request_servername () {
for i in `dig +short name.cloudfront.net`; do
elog "Check ip: ${i}" >> script.log
servername="$(curl -s -o /dev/null -k --connect-timeout 5 -w "%{http_code};%{time_total};%{time_connect};%{time_appconnect};%{time_starttransfer};%{time_namelookup}\n" -H "Host: name.cloudfront.net" https://${i} | tee -a script.log"
if [[ "${servername}" != 200* ]]; then
mtr --tcp --port 443 -Z 10 -c 10 -r ${i} >> script.log
fi
done
}
main "$@"
script.log
2023-10-20 01:46:27 Start server.name checking
code;time_total;time_connect;time_appconnect;time_starttransfer;time_namelookup
2023-10-20 01:46:27 Check ip: 18.66.92.206
200;0.029263;0.002201;0.019988;0.026997;0.000030
2023-10-20 01:46:27 Check ip: 18.66.92.222
200;0.028911;0.004471;0.020986;0.026301;0.000036
2023-10-20 01:46:27 Check ip: 18.66.92.120
200;0.032083;0.002433;0.023399;0.029845;0.000138
2023-10-20 01:46:27 Check ip: 18.66.92.89
200;0.031340;0.002665;0.022837;0.028826;0.000037
2023-10-20 01:46:27 End server.name checking
2023-10-20 01:47:28 Start server.name checking
code;time_total;time_connect;time_appconnect;time_starttransfer;time_namelookup
2023-10-20 01:47:28 Check ip: 18.66.92.222
200;0.028406;0.003000;0.018684;0.025747;0.000029
2023-10-20 01:47:28 Check ip: 18.66.92.120
200;0.027571;0.002188;0.018924;0.025329;0.000029
2023-10-20 01:47:28 Check ip: 18.66.92.206
200;0.026843;0.002167;0.018254;0.024549;0.000031
2023-10-20 01:47:28 Check ip: 18.66.92.89
200;0.038611;0.002388;0.022507;0.036310;0.000032
2023-10-20 01:47:28 End server.name checking
2023-10-20 01:48:28 Start server.name checking
code;time_total;time_connect;time_appconnect;time_starttransfer;time_namelookup
2023-10-20 01:48:28 Check ip: 18.66.92.120
200;0.031731;0.002783;0.023693;0.029298;0.000036
2023-10-20 01:48:28 Check ip: 18.66.92.206
200;0.034007;0.002266;0.025089;0.031776;0.000081
2023-10-20 01:48:28 Check ip: 18.66.92.222
200;0.046491;0.002205;0.035124;0.043846;0.000039
2023-10-20 01:48:28 Check ip: 18.66.92.89
200;0.034650;0.002135;0.025856;0.032022;0.000056
2023-10-20 01:48:28 End server.name checking
As you can see at the same time openresty wrong resolve cloudfront name and throw error.
If the problem reoccurs, you can use tcpdump to grab packets and analyze the DNS response.
If there is no response, then the message that should have been a dns request is dropped by the network link. This should happen because the dns quaternion remains the same for a long time, but your source IP address changes and other similar factors cause the message to be dropped by firewalls, NATs, and other network components.