The engine plan file is not compatible
felixdollack opened this issue · comments
Trying to run HEAD / 3ad8cea leads to the following traceback:
face_1 | [05/02/2022-15:17:45] [TRT] [E] 6: The engine plan file is not compatible with this version of TensorRT, expecting library version 8.2.3 got 8.2.1, please rebuild.
face_1 | [05/02/2022-15:17:45] [TRT] [E] 4: [runtime.cpp::deserializeCudaEngine::50] Error Code 4: Internal Error (Engine deserialization failed.)
face_1 | Traceback (most recent call last):
face_1 | File "/usr/local/bin/uvicorn", line 8, in <module>
face_1 | sys.exit(main())
face_1 | File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1130, in __call__
face_1 | return self.main(*args, **kwargs)
face_1 | File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1055, in main
face_1 | rv = self.invoke(ctx)
face_1 | File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 1404, in invoke
face_1 | return ctx.invoke(self.callback, **ctx.params)
face_1 | File "/usr/local/lib/python3.8/dist-packages/click/core.py", line 760, in invoke
face_1 | return __callback(*args, **kwargs)
face_1 | File "/usr/local/lib/python3.8/dist-packages/uvicorn/main.py", line 437, in main
face_1 | run(app, **kwargs)
face_1 | File "/usr/local/lib/python3.8/dist-packages/uvicorn/main.py", line 463, in run
face_1 | server.run()
face_1 | File "/usr/local/lib/python3.8/dist-packages/uvicorn/server.py", line 60, in run
face_1 | return asyncio.run(self.serve(sockets=sockets))
face_1 | File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
face_1 | return loop.run_until_complete(main)
face_1 | File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
face_1 | return future.result()
face_1 | File "/usr/local/lib/python3.8/dist-packages/uvicorn/server.py", line 67, in serve
face_1 | config.load()
face_1 | File "/usr/local/lib/python3.8/dist-packages/uvicorn/config.py", line 458, in load
face_1 | self.loaded_app = import_from_string(self.app)
face_1 | File "/usr/local/lib/python3.8/dist-packages/uvicorn/importer.py", line 21, in import_from_string
face_1 | module = importlib.import_module(module_str)
face_1 | File "/usr/lib/python3.8/importlib/__init__.py", line 127, in import_module
face_1 | return _bootstrap._gcd_import(name[level:], package, level)
face_1 | File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
face_1 | File "<frozen importlib._bootstrap>", line 991, in _find_and_load
face_1 | File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
face_1 | File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
face_1 | File "<frozen importlib._bootstrap_external>", line 848, in exec_module
face_1 | File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
face_1 | File "/app/./app.py", line 36, in <module>
face_1 | processing = Processing(det_name=configs.models.det_name, rec_name=configs.models.rec_name,
face_1 | File "/app/./modules/processing.py", line 94, in __init__
face_1 | self.model = FaceAnalysis(det_name=det_name, rec_name=rec_name, ga_name=ga_name,
face_1 | File "/app/./modules/face_model.py", line 81, in __init__
face_1 | self.det_model = Detector(det_name=det_name, device=device, max_size=self.max_size,
face_1 | File "/app/./modules/face_model.py", line 41, in __init__
face_1 | self.retina.prepare(ctx_id=device2ctx[device], nms=0.35)
face_1 | File "/app/./modules/model_zoo/detectors/scrfd.py", line 177, in prepare
face_1 | self.session.prepare()
face_1 | File "/app/./modules/model_zoo/exec_backends/trt_backend.py", line 174, in prepare
face_1 | self.rec_model.build()
face_1 | File "/app/./modules/model_zoo/exec_backends/trt_loader.py", line 67, in build
face_1 | raise RuntimeError(f'Unable to load the engine file {self.engine_file}')
face_1 | RuntimeError: Unable to load the engine file /models/trt-engines/scrfd_10g_gnkps/scrfd_10g_gnkps_640_640.plan
I modified trt_loader.py to print the model-filename.
As the first line in the traceback says, expecting library version 8.2.3 got 8.2.1, please rebuild.
I tried with the default model /models/trt-engines/scrfd_10g_gnkps/scrfd_10g_gnkps_640_640.plan running the TensorRT version.
You should just remove trt-engines dir and let container build new engines.