Exception raised when reloading quickly multiple times - error handling missing
MarcSkovMadsen opened this issue · comments
panel main branch (post 1.4.1).
It seems the .remove(module) line below is missing error handling.
Reproduce
import threading
import time
import cv2 as cv
import param
from PIL import Image
from PIL import Image, ImageDraw, ImageFont
import logging
import panel as pn
pn.extension()
class CannotOpenCamera(Exception):
"""Exception raised if the camera cannot be opened."""
class CannotReadCamera(Exception):
"""Exception raised if the camera cannot be read."""
class ServerVideoStream(pn.viewable.Viewer):
value = param.Parameter(doc="The current snapshot as a Pillow Image")
exception = param.ClassSelector(class_=Exception, allow_None=True, doc="""The last exception or None""")
status = param.String()
paused = param.Boolean(default=False, doc="Whether the video stream is paused")
fps = param.Number(10, doc="Target frames per second", inclusive_bounds=(0, None))
min_sleep = param.Number(0.1, doc="""The minimum time to sleep between each frame capture.
Adjust this to keep your app responsive based on your specific use case.""")
camera_index = param.Integer(0, doc="The index of the active camera")
log_level = param.Selector(default=logging.DEBUG, objects=[logging.DEBUG, logging.WARNING])
_cache_key = "__server_video_stream__"
def __init__(self, **params):
print("Init", params)
self._empty_image = self._create_placeholder_image()
params["value"]=params.get("value", self._empty_image)
super().__init__(**params)
self._cameras = {}
self._stop_thread = False
self._thread = threading.Thread(target=self._take_images, daemon=True)
self._thread.start()
def __new__(cls, **params):
print("New", params)
stream = pn.state.cache.get(cls._cache_key)
if not stream:
print("new")
stream = super(ServerVideoStream, cls).__new__(cls)
pn.state.cache[cls._cache_key] = stream
else:
stream.param.update(**params)
return stream
def _log(self, message):
"""Log a message"""
if self.log_level==logging.DEBUG and message!=self.status:
print(f"{id(self)} - {message}")
self.status = message
def _remove_core(self):
"""Securely stops and removes the ServerVideoStream."""
self._log("Removing")
self._stop_thread = True
if self._thread.is_alive():
self._thread.join(1.0)
for camera in self._cameras.values():
camera.release()
cv.destroyAllWindows()
if self._cache_key in pn.state.cache:
del pn.state.cache[self._cache_key]
self._log("Removed")
@classmethod
def remove(cls):
"""Removes the current VideoStream"""
stream = pn.state.cache.pop(cls._cache_key, None)
if stream:
stream._remove_core()
def get_camera(self, index):
if index in self._cameras:
return self._cameras[index]
self._log(f"Getting Camera {index}")
cap = cv.VideoCapture(index)
if not cap.isOpened():
raise CannotOpenCamera(f"Cannot open the camera {index}")
self._cameras[index] = cap
return cap
@staticmethod
def _cv2_to_pil(bgr_image):
rgb_image = cv.cvtColor(bgr_image, cv.COLOR_BGR2RGB)
image = Image.fromarray(rgb_image)
return image
@staticmethod
def _create_placeholder_image(bg_color="#CCCCCC", text="No Frame Available"):
width, height = 500, 500
image = Image.new("RGB", (width, height), bg_color)
draw = ImageDraw.Draw(image)
font = ImageFont.load_default(int(height/20))
draw.text((150, 225), text, font=font, fill="black")
return image
def _take_image(self):
camera = self.get_camera(self.camera_index)
self._log(f"Reading camera {self.camera_index}")
ret, frame = camera.read()
if not ret:
raise CannotReadCamera("Ensure the camera exists and is not in use by other processes.")
else:
self.value = self._cv2_to_pil(frame)
def _take_images(self):
while not self._stop_thread:
start_time = time.time()
self._log("paused: "+str(self.paused))
if not self.paused:
try:
self._take_image()
self.exception=None
except Exception as ex:
self.value=self._empty_image
self.exception=ex
else:
self._log("Paused")
if self.fps > 0:
interval = 1 / self.fps
elapsed_time = time.time() - start_time
sleep_time = max(0.1, interval - elapsed_time)
self._log(f"Sleeping {sleep_time}")
time.sleep(sleep_time)
def __panel__(self):
title = f"## VideoStream {id(self)}"
settings = pn.Column(
title,
self.param.paused,
self.param.fps,
self.param.camera_index,
pn.widgets.TextInput(value=self.param.status, disabled=True, name="Status"),
pn.widgets.LiteralInput(value=self.param.exception, disabled=True, name="Exception"),
)
image = pn.pane.Image(self.param.value, sizing_mode="stretch_both")
return pn.Row(settings, image)
stream=ServerVideoStream(fps=60, camera_index=1, paused=False, log_level=logging.WARNING)
print(stream.paused)
stream=ServerVideoStream(fps=60, camera_index=1, paused=True, log_level=logging.WARNING)
print(stream.paused)
stream.servable()- Serve the app with autoreload
- Reload the app several times quickly after each other
- See the exception raised one or more times in the terminal.
HTTPServerRequest(protocol='http', host='localhost:5006', method='GET', uri='/script', version='HTTP/1.1', remote_ip='::1')
Traceback (most recent call last):
File "c:\repos\private\panel\panel\io\handlers.py", line 389, in run
post_check()
File "c:\repos\private\panel\panel\io\handlers.py", line 258, in post_check
raise RuntimeError("%s at '%s' replaced the output document" % (handler._origin, handler._runner.path))
RuntimeError: Script at 'C:\repos\private\panel\script.py' replaced the output document
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\repos\private\panel\.venv\Lib\site-packages\tornado\web.py", line 1790, in _execute
result = await result
^^^^^^^^^^^^
File "c:\repos\private\panel\panel\io\server.py", line 527, in get
session = await self.get_session()
^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\repos\private\panel\panel\io\server.py", line 416, in get_session
session = await super().get_session()
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\server\views\session_handler.py", line 145, in get_session
session = await self.application_context.create_session_if_needed(session_id, self.request, token)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\server\contexts.py", line 240, in create_session_if_needed
self._application.initialize_document(doc)
File "c:\repos\private\panel\panel\io\application.py", line 76, in initialize_document
super().initialize_document(doc)
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\application\application.py", line 190, in initialize_document
h.modify_document(doc)
File "c:\repos\private\panel\panel\io\handlers.py", line 442, in modify_document
run_app(self, module, doc)
File "c:\repos\private\panel\panel\io\handlers.py", line 266, in run_app
handler._runner.run(module, post_check)
File "c:\repos\private\panel\panel\io\handlers.py", line 391, in run
autoreload_handle_exception(self, module, e)
File "c:\repos\private\panel\panel\io\handlers.py", line 229, in autoreload_handle_exception
state.curdoc.modules._modules.remove(module)
ValueError: list.remove(x): x not in listmissing-error-handling.mp4
I tried adding a try except. But the I started seeing other mysterious exception.
I removed the try except again.
When having multiple windows open and reloading one of them I also see
2024-04-21 07:27:21,401 Error running application handler <panel.io.handlers.ScriptHandler object at 0x0000019460880150>: Script at 'C:\repos\private\panel\script.py' replaced the output document
File 'handlers.py', line 258, in post_check:
raise RuntimeError("%s at '%s' replaced the output document" % (handler._origin, handler._runner.path)) Traceback (most recent call last):
File "c:\repos\private\panel\panel\io\handlers.py", line 389, in run
post_check()
File "c:\repos\private\panel\panel\io\handlers.py", line 258, in post_check
raise RuntimeError("%s at '%s' replaced the output document" % (handler._origin, handler._runner.path))
RuntimeError: Script at 'C:\repos\private\panel\script.py' replaced the output document
2024-04-21 07:27:21,494 WebSocket connection closed: code=1001, reason=None
2024-04-21 07:27:21,528 Failed sending message as connection was closed
2024-04-21 07:27:21,530 WebSocket connection opened
2024-04-21 07:27:21,568 ServerConnection created
2024-04-21 07:27:51,074 Error running application handler <panel.io.handlers.ScriptHandler object at 0x0000019460880150>: _pending_writes should be non-None when we have a document lock, and we should have the lock when the document changes
File 'session.py', line 244, in _document_patched:
raise RuntimeError("_pending_writes should be non-None when we have a document lock, and we should have the lock when the document changes") Traceback (most recent call last):
File "c:\repos\private\panel\panel\io\handlers.py", line 386, in run
exec(self._code, module.__dict__)
File "C:\repos\private\panel\script.py", line 186, in <module>
stream.servable()
File "c:\repos\private\panel\panel\viewable.py", line 1054, in servable
return self._create_view().servable(title, location, area, target)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\repos\private\panel\panel\viewable.py", line 395, in servable
self.server_doc(title=title, location=location) # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\repos\private\panel\panel\viewable.py", line 1011, in server_doc
add_to_doc(model, doc)
File "c:\repos\private\panel\panel\io\model.py", line 118, in add_to_doc
doc.add_root(obj)
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\document\document.py", line 324, in add_root
self.callbacks.trigger_on_change(RootAddedEvent(self, model, setter))
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\document\callbacks.py", line 413, in trigger_on_change
invoke_with_curdoc(doc, invoke_callbacks)
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\document\callbacks.py", line 443, in invoke_with_curdoc
return f()
^^^
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\document\callbacks.py", line 412, in invoke_callbacks
cb(event)
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\document\callbacks.py", line 276, in <lambda>
self._change_callbacks[receiver] = lambda event: event.dispatch(receiver)
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\document\events.py", line 219, in dispatch
cast(DocumentPatchedMixin, receiver)._document_patched(self)
File "C:\repos\private\panel\.venv\Lib\site-packages\bokeh\server\session.py", line 244, in _document_patched
raise RuntimeError("_pending_writes should be non-None when we have a document lock, and we should have the lock when the document changes")
RuntimeError: _pending_writes should be non-None when we have a document lock, and we should have the lock when the document changesAlso if I have a large number of windows open (4), set the fps high (60) and the min_sleep low (0.1). I can also be hard to pause the camera via the checkbox. The camera loop just keeps running for a long time sending the frames to the frontend
Later it sends the paused checkbox to all the open windows and pauses the video.
stream=ServerVideoStream(fps=60, camera_index=1, min_sleep=0.1, paused=False, log_level=logging.WARNING)Its like the separate thread loop is preferred over the main thread.
When having multiple windows open and reloading one of them I also see
That's a little scary and needs a new issue. It's likely something to do with the threads.
Also if I have a large number of windows open (4), set the fps high (60) and the min_sleep low (0.1). I can also be hard to pause the camera via the checkbox.
We can consider somehow throttling events, the problem here is that you're effectively queuing up a bunch of events that have to be sent across the websocket and they all pile up. So even when you turn off the spigot the backlog still needs to be processed.