Parent process logging SoftTimeout
alanhamlett opened this issue · comments
Received this error in logs yesterday, but it's strange because the Parent process should never execute Worker._on_soft_timeout_child. That method should only execute on Child processes when the Parent process sends a SoftTimeout via SIGQUIT to the child. The Parent process executes Worker._on_exit_parent when it receives SIGQUIT.
full traceback
[2023-12-17 09:29:39,088] worker-03 ERROR in None args=None kwargs=None retry=None: Unable to unpack message from child process 1434680: backup_heartbeat
[2023-12-17 09:29:39,111] worker-03 ERROR in None args=None kwargs=None retry=None: Unable to unpack message from child process 1434689: backup_he
[2023-12-17 09:29:39,111] worker-03 ERROR in None args=None kwargs=None retry=None: Unable to unpack message from child process 1434864: backup_
[2023-12-17 09:33:46,210] worker-03 ERROR in None args=None kwargs=None retry=None: Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 192, in _parent
self._check_child_runtimes()
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 520, in _check_child_runtimes
elif not child.soft_timeout_reached and soft_timeout and runtime > soft_timeout:
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 358, in _on_soft_timeout_child
raise SoftTimeout("SoftTimeout")
wakaq.exceptions.SoftTimeout: SoftTimeout
[2023-12-17 09:33:48,849] worker-03 ERROR in None args=None kwargs=None retry=None: Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/utils.py", line 151, in read_fd
return os.read(fd, 64000).decode("utf8")
BlockingIOError: [Errno 11] Resource temporarily unavailable
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 192, in _parent
self._check_child_runtimes()
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 496, in _check_child_runtimes
ping = read_fd(child.pingin)
File "./venv/lib/python3.10/site-packages/wakaq/utils.py", line 151, in read_fd
return os.read(fd, 64000).decode("utf8")
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 358, in _on_soft_timeout_child
raise SoftTimeout("SoftTimeout")
wakaq.exceptions.SoftTimeout: SoftTimeout
[2023-12-17 09:33:51,138] worker-03 ERROR in None args=None kwargs=None retry=None: Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 183, in _parent
self._enqueue_ready_eta_tasks()
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 385, in _enqueue_ready_eta_tasks
self.wakaq._enqueue_at_front(task_name, queue.name, args, kwargs)
File "./venv/lib/python3.10/site-packages/wakaq/__init__.py", line 189, in _enqueue_at_front
payload = serialize(
File "./venv/lib/python3.10/site-packages/wakaq/serializer.py", line 93, in serialize
return dumps(*args, **kwargs)
File "/usr/lib/python3.10/json/__init__.py", line 238, in dumps
**kw).encode(obj)
File "/usr/lib/python3.10/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/lib/python3.10/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 358, in _on_soft_timeout_child
raise SoftTimeout("SoftTimeout")
wakaq.exceptions.SoftTimeout: SoftTimeout
[2023-12-17 09:33:51,126] worker-03 ERROR in None args=None kwargs=None retry=None: Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/utils.py", line 151, in read_fd
return os.read(fd, 64000).decode("utf8")
BlockingIOError: [Errno 11] Resource temporarily unavailable
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 192, in _parent
self._check_child_runtimes()
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 496, in _check_child_runtimes
ping = read_fd(child.pingin)
File "./venv/lib/python3.10/site-packages/wakaq/utils.py", line 151, in read_fd
return os.read(fd, 64000).decode("utf8")
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 358, in _on_soft_timeout_child
raise SoftTimeout("SoftTimeout")
wakaq.exceptions.SoftTimeout: SoftTimeout
This traceback in particular doesn't make sense, and shouldn't be possible:
Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 192, in _parent
self._check_child_runtimes()
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 496, in _check_child_runtimes
ping = read_fd(child.pingin)
File "./venv/lib/python3.10/site-packages/wakaq/utils.py", line 151, in read_fd
return os.read(fd, 64000).decode("utf8")
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 358, in _on_soft_timeout_child
raise SoftTimeout("SoftTimeout")
wakaq.exceptions.SoftTimeout: SoftTimeout
another traceback
[2023-12-23 07:55:11,392] wakaq-worker-04 ERROR in None args=None kwargs=None retry=None: Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 186, in _parent
self._listen_for_broadcast_task()
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 526, in _listen_for_broadcast_task
msg = self._pubsub.get_message(ignore_subscribe_messages=True, timeout=self.wakaq.wait_timeout)
File "./venv/lib/python3.10/site-packages/redis/client.py", line 1013, in get_message
response = self.parse_response(block=(timeout is None), timeout=timeout)
File "./venv/lib/python3.10/site-packages/redis/client.py", line 824, in parse_response
response = self._execute(conn, try_read)
File "./venv/lib/python3.10/site-packages/redis/client.py", line 800, in _execute
return conn.retry.call_with_retry(
File "./venv/lib/python3.10/site-packages/redis/retry.py", line 46, in call_with_retry
return do()
File "./venv/lib/python3.10/site-packages/redis/client.py", line 801, in <lambda>
lambda: command(*args, **kwargs),
File "./venv/lib/python3.10/site-packages/redis/client.py", line 818, in try_read
if not conn.can_read(timeout=timeout):
File "./venv/lib/python3.10/site-packages/redis/connection.py", line 478, in can_read
return self._parser.can_read(timeout)
File "./venv/lib/python3.10/site-packages/redis/_parsers/base.py", line 128, in can_read
return self._buffer and self._buffer.can_read(timeout)
File "./venv/lib/python3.10/site-packages/redis/_parsers/socket.py", line 95, in can_read
return bool(self.unread_bytes()) or self._read_from_socket(
File "./venv/lib/python3.10/site-packages/redis/_parsers/socket.py", line 65, in _read_from_socket
data = self._sock.recv(socket_read_size)
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 358, in _on_soft_timeout_child
raise SoftTimeout("SoftTimeout")
wakaq.exceptions.SoftTimeout: SoftTimeout
and another traceback
[2023-12-23 07:55:11,393] wakaq-worker-04 ERROR in None args=None kwargs=None retry=None: Traceback (most recent call last):
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 183, in _parent
self._enqueue_ready_eta_tasks()
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 379, in _enqueue_ready_eta_tasks
results = script(keys=[queue.broker_eta_key], args=[int(round(time.time()))])
File "./venv/lib/python3.10/site-packages/redis/commands/core.py", line 5931, in __call__
return client.evalsha(self.sha, len(keys), *args)
File "./venv/lib/python3.10/site-packages/redis/commands/core.py", line 5307, in evalsha
return self._evalsha("EVALSHA", sha, numkeys, *keys_and_args)
File "./venv/lib/python3.10/site-packages/redis/commands/core.py", line 5291, in _evalsha
return self.execute_command(command, sha, numkeys, *keys_and_args)
File "./venv/lib/python3.10/site-packages/redis/client.py", line 536, in execute_command
return conn.retry.call_with_retry(
File "./venv/lib/python3.10/site-packages/redis/retry.py", line 46, in call_with_retry
return do()
File "./venv/lib/python3.10/site-packages/redis/client.py", line 537, in <lambda>
lambda: self._send_command_parse_response(
File "./venv/lib/python3.10/site-packages/redis/client.py", line 513, in _send_command_parse_response
return self.parse_response(conn, command_name, **options)
File "./venv/lib/python3.10/site-packages/redis/client.py", line 553, in parse_response
response = connection.read_response()
File "./venv/lib/python3.10/site-packages/redis/connection.py", line 500, in read_response
response = self._parser.read_response(disable_decoding=disable_decoding)
File "./venv/lib/python3.10/site-packages/redis/_parsers/resp2.py", line 15, in read_response
result = self._read_response(disable_decoding=disable_decoding)
File "./venv/lib/python3.10/site-packages/redis/_parsers/resp2.py", line 25, in _read_response
raw = self._buffer.readline()
File "./venv/lib/python3.10/site-packages/redis/_parsers/socket.py", line 115, in readline
self._read_from_socket()
File "./venv/lib/python3.10/site-packages/redis/_parsers/socket.py", line 65, in _read_from_socket
data = self._sock.recv(socket_read_size)
File "./venv/lib/python3.10/site-packages/wakaq/worker.py", line 358, in _on_soft_timeout_child
raise SoftTimeout("SoftTimeout")
wakaq.exceptions.SoftTimeout: SoftTimeout