stefanprodan / k8s-prom-hpa

Kubernetes Horizontal Pod Autoscaler with Prometheus custom metrics

Geek Repo:Geek Repo

Github PK Tool:Github PK Tool

metrics-server v1beta1.metrics.k8s.io failed : net/http: request canceled while waiting for connection

fengjian1585 opened this issue · comments

    1. kubectl create -f ./metrics-server

master1 kube-apiserver: E0412 22:18:30.255424 2628 available_controller.go:295] v1beta1.metrics.k8s.io failed with: Get https://10.233.53.4:443: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)

    1. listen port

[root@master1 1.8+]# kubectl exec -it metrics-server-7bcc5bf8f-pk865 -n kube-system sh / # ps -ef PID USER TIME COMMAND 1 root 0:00 /metrics-server --source=kubernetes.summary_api:'' --requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem 38 root 0:00 sh 43 root 0:00 ps -ef / # netstat -anpt Active Internet connections (servers and established) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 10.233.53.4:49394 10.254.0.1:443 ESTABLISHED 1/metrics-server tcp 0 0 10.233.53.4:52990 192.168.200.52:10255 ESTABLISHED 1/metrics-server tcp 0 0 10.233.53.4:51762 192.168.200.53:10255 ESTABLISHED 1/metrics-server tcp 0 0 :::443 :::* LISTEN 1/metrics-server / #

    1. metrics-server log

[root@master1 1.8+]# kubectl logs -f metrics-server-7bcc5bf8f-pk865 -n kube-system I0412 14:15:02.347895 1 heapster.go:71] /metrics-server --source=kubernetes.summary_api:'' --requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem I0412 14:15:02.347957 1 heapster.go:72] Metrics Server version v0.2.1 I0412 14:15:02.348159 1 configs.go:61] Using Kubernetes client with master "https://10.254.0.1:443" and version I0412 14:15:02.348194 1 configs.go:62] Using kubelet port 10255 I0412 14:15:02.349072 1 heapster.go:128] Starting with Metric Sink I0412 14:15:02.555531 1 serving.go:308] Generated self-signed cert (apiserver.local.config/certificates/apiserver.crt, apiserver.local.config/certificates/apiserver.key) I0412 14:15:02.933037 1 heapster.go:101] Starting Heapster API server... [restful] 2018/04/12 14:15:02 log.go:33: [restful/swagger] listing is available at https:///swaggerapi [restful] 2018/04/12 14:15:02 log.go:33: [restful/swagger] https:///swaggerui/ is mapped to folder /swagger-ui/ I0412 14:15:02.934173 1 serve.go:85] Serving securely on 0.0.0.0:443

    1. kube-apiserver config

[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
User=root
ExecStart=/usr/local/bin/kube-apiserver
--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
--advertise-address=192.168.200.51
--allow-privileged=true
--anonymous-auth=false
--apiserver-count=1
--audit-policy-file=/etc/kubernetes/audit-policy.yaml
--audit-log-maxage=30
--audit-log-maxbackup=3
--audit-log-maxsize=100
--audit-log-path=/var/log/kubernetes/audit.log
--authorization-mode=Node,RBAC
--bind-address=0.0.0.0
--secure-port=6443
--client-ca-file=/etc/kubernetes/ssl/ca.pem
--enable-swagger-ui=true
--etcd-cafile=/etc/kubernetes/ssl/ca.pem
--etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem
--etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem
--etcd-servers=https://192.168.200.51:2379,https://192.168.200.52:2379,https://192.168.200.53:2379
--event-ttl=1h
--kubelet-https=true
--insecure-bind-address=192.168.200.51
--insecure-port=8080
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem
--service-cluster-ip-range=10.254.0.0/16
--service-node-port-range=30000-32000
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem
--enable-bootstrap-token-auth=true
--token-auth-file=/etc/kubernetes/token.csv
--requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem
--proxy-client-cert-file=/etc/kubernetes/ssl/front-proxy-client.pem
--proxy-client-key-file=/etc/kubernetes/ssl/front-proxy-client-key.pem
--requestheader-allowed-names=aggregator
--requestheader-group-headers=X-Remote-Group
--requestheader-extra-headers-prefix=X-Remote-Extra-
--requestheader-username-headers=X-Remote-User
--runtime-config=admissionregistration.k8s.io/v1alpha1
--runtime-config=api/all=true
--enable-aggregator-routing=true
--v=0
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

    1. kube-controller-manager config

[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
ExecStart=/usr/local/bin/kube-controller-manager
--address=0.0.0.0
--master=http://192.168.200.51:8080
--service-cluster-ip-range=10.254.0.0/16
--cluster-name=kubernetes
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem
--root-ca-file=/etc/kubernetes/ssl/ca.pem
--leader-elect=true
--v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

Same issue: v1beta1.metrics.k8s.io failed with: Get https://10.200.4.7:443: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
Any one can help?