Running chaos tests fails on OCP 4.11 nightly builds
pravin-dsilva opened this issue · comments
I see the following errors while running Kraken on an Openshift Container Platform nightly build of version 4.11. The same command runs fine on OCP 4.10. I have tried this with 2 other 4.11 clusters and see the same error.
# oc version
Client Version: 4.11.0-0.nightly-ppc64le-2022-04-28-084330
Kustomize Version: v4.5.4
Server Version: 4.11.0-0.nightly-ppc64le-2022-04-28-084330
Kubernetes Version: v1.23.3+d464c70
Kraken logs:
# docker logs kraken -f
2022-05-09 10:58:39,938 [INFO] Starting kraken
2022-05-09 10:58:39,953 [INFO] Initializing client to talk to the Kubernetes cluster
2022-05-09 11:01:18,354 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection broken: IncompleteRead(5130112 bytes read)', IncompleteRead(5130112 bytes read))': /api/v1/pods?pretty=True
2022-05-09 11:03:55,208 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection broken: IncompleteRead(5130497 bytes read)', IncompleteRead(5130497 bytes read))': /api/v1/pods?pretty=True
2022-05-09 11:06:35,219 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection broken: IncompleteRead(5130072 bytes read)', IncompleteRead(5130072 bytes read))': /api/v1/pods?pretty=True
Traceback (most recent call last):
File "/usr/lib64/python3.9/http/client.py", line 560, in _get_chunk_left
_ _
| | ___ __ __ _| | _____ _ __
| |/ / '__/ _` | |/ / _ \ '_ \
| <| | | (_| | < __/ | | |
|_|\_\_| \__,_|_|\_\___|_| |_|
chunk_left = self._read_next_chunk_size()
File "/usr/lib64/python3.9/http/client.py", line 527, in _read_next_chunk_size
return int(line, 16)
ValueError: invalid literal for int() with base 16: b''
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib64/python3.9/http/client.py", line 577, in _readall_chunked
chunk_left = self._get_chunk_left()
File "/usr/lib64/python3.9/http/client.py", line 562, in _get_chunk_left
raise IncompleteRead(b'')
http.client.IncompleteRead: IncompleteRead(0 bytes read)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 441, in _error_catcher
yield
File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 518, in read
data = self._fp.read() if not fp_closed else b""
File "/usr/lib64/python3.9/http/client.py", line 470, in read
return self._readall_chunked()
File "/usr/lib64/python3.9/http/client.py", line 584, in _readall_chunked
raise IncompleteRead(b''.join(value))
http.client.IncompleteRead: IncompleteRead(5130072 bytes read)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 723, in urlopen
response = self.ResponseCls.from_httplib(
File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 604, in from_httplib
resp = ResponseCls(
File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 263, in __init__
self._body = self.read(decode_content=decode_content)
File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 544, in read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
File "/usr/lib64/python3.9/contextlib.py", line 137, in __exit__
self.gen.throw(typ, value, traceback)
File "/usr/local/lib/python3.9/site-packages/urllib3/response.py", line 458, in _error_catcher
raise ProtocolError("Connection broken: %r" % e, e)
urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(5130072 bytes read)', IncompleteRead(5130072 bytes read))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root/kraken/run_kraken.py", line 309, in <module>
main(options.cfg)
File "/root/kraken/run_kraken.py", line 82, in main
kubecli.find_kraken_node()
File "/root/kraken/kraken/kubernetes/client.py", line 366, in find_kraken_node
pods = get_all_pods()
File "/root/kraken/kraken/kubernetes/client.py", line 135, in get_all_pods
ret = cli.list_pod_for_all_namespaces(pretty=True)
File "/usr/local/lib/python3.9/site-packages/kubernetes/client/api/core_v1_api.py", line 16864, in list_pod_for_all_namespaces
return self.list_pod_for_all_namespaces_with_http_info(**kwargs) # noqa: E501
File "/usr/local/lib/python3.9/site-packages/kubernetes/client/api/core_v1_api.py", line 16967, in list_pod_for_all_namespaces_with_http_info
return self.api_client.call_api(
File "/usr/local/lib/python3.9/site-packages/kubernetes/client/api_client.py", line 348, in call_api
return self.__call_api(resource_path, method,
File "/usr/local/lib/python3.9/site-packages/kubernetes/client/api_client.py", line 180, in __call_api
response_data = self.request(
File "/usr/local/lib/python3.9/site-packages/kubernetes/client/api_client.py", line 373, in request
return self.rest_client.GET(url,
File "/usr/local/lib/python3.9/site-packages/kubernetes/client/rest.py", line 239, in GET
return self.request("GET", url,
File "/usr/local/lib/python3.9/site-packages/kubernetes/client/rest.py", line 212, in request
r = self.pool_manager.request(method, url,
File "/usr/local/lib/python3.9/site-packages/urllib3/request.py", line 74, in request
return self.request_encode_url(
File "/usr/local/lib/python3.9/site-packages/urllib3/request.py", line 96, in request_encode_url
return self.urlopen(method, url, **extra_kw)
File "/usr/local/lib/python3.9/site-packages/urllib3/poolmanager.py", line 376, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 813, in urlopen
return self.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 813, in urlopen
return self.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 813, in urlopen
return self.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 785, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='api.pravinp10-0428.redhat.com', port=6443): Max retries exceeded with url: /api/v1/pods?pretty=True (Caused by ProtocolError('Connection broken: IncompleteRead(5130072 bytes read)', IncompleteRead(5130072 bytes read)))
This was happening only on 2 of my clusters and was able to run on some other 4.11 clusters. I was able to proceed with the chaos tests so closing this.