Errors in unit test
mxochicale opened this issue · comments
The following test produce errors
test_charuco_plus_chessboard.py
test_chessboard_calibration.py
test_hand_eye.py
test_precalib.py
logs for test_precalib.py:
==================================================================================================== FAILURES =====================================================================================================
_____________________________________________________________________________________________ test_charuco_dataset_B ______________________________________________________________________________________________
def test_charuco_dataset_B():
calib_dir = 'tests/data/precalib/data_moved_scope'
calib_driver = get_calib_driver(calib_dir)
stereo_reproj_err, stereo_recon_err, _ = \
calib_driver.calibrate()
tracked_reproj_err, tracked_recon_err, _ = \
calib_driver.handeye_calibration()
print(stereo_reproj_err, stereo_recon_err, tracked_reproj_err, tracked_recon_err)
> assert stereo_reproj_err < 1
E assert 7.116922375480872 < 1
tests/video/test_precalib.py:92: AssertionError
_______________________________________________________________________________________________ test_precalbration ________________________________________________________________________________________________
def test_precalbration():
""" Use intrinsics from A to calibration B, currently failing. """
left_intrinsics = np.loadtxt('tests/data/precalib/precalib_base_data/calib.left.intrinsics.txt')
left_distortion = np.loadtxt('tests/data/precalib/precalib_base_data/calib.left.distortion.txt')
right_intrinsics = np.loadtxt('tests/data/precalib/precalib_base_data/calib.right.intrinsics.txt')
right_distortion = np.loadtxt('tests/data/precalib/precalib_base_data/calib.right.distortion.txt')
l2r = np.loadtxt('tests/data/precalib/precalib_base_data/calib.l2r.txt')
l2r_rmat = l2r[0:3, 0:3]
l2r_tvec = l2r[0:3, 3]
calib_dir = 'tests/data/precalib/data_moved_scope'
calib_driver = get_calib_driver(calib_dir)
stereo_reproj_err, stereo_recon_err, _ = \
calib_driver.calibrate(
override_left_intrinsics=left_intrinsics,
override_left_distortion=left_distortion,
override_right_intrinsics=right_intrinsics,
override_right_distortion=right_distortion,
override_l2r_rmat=l2r_rmat,
override_l2r_tvec=l2r_tvec)
tracked_reproj_err, tracked_recon_err, _ = \
calib_driver.handeye_calibration()
print(stereo_reproj_err, stereo_recon_err, tracked_reproj_err, tracked_recon_err)
> assert stereo_reproj_err < 4.5
E assert 34.74794862170866 < 4.5
tests/video/test_precalib.py:124: AssertionError
================================================================================================ warnings s
Next steps
Testing test_precalib.py with local conda env using opencv-contrib-python-headless 4.5.5.64 pypi_0 pypi
==================================================================================================== FAILURES =====================================================================================================
_____________________________________________________________________________________________ test_charuco_dataset_B ______________________________________________________________________________________________
def test_charuco_dataset_B():
calib_dir = 'tests/data/precalib/data_moved_scope'
calib_driver = get_calib_driver(calib_dir)
stereo_reproj_err, stereo_recon_err, _ = \
calib_driver.calibrate()
tracked_reproj_err, tracked_recon_err, _ = \
calib_driver.handeye_calibration()
print(stereo_reproj_err, stereo_recon_err, tracked_reproj_err, tracked_recon_err)
> assert stereo_reproj_err < 1
E assert 7.116922375480872 < 1
tests/video/test_precalib.py:92: AssertionError
_______________________________________________________________________________________________ test_precalbration ________________________________________________________________________________________________
def test_precalbration():
""" Use intrinsics from A to calibration B, currently failing. """
left_intrinsics = np.loadtxt('tests/data/precalib/precalib_base_data/calib.left.intrinsics.txt')
left_distortion = np.loadtxt('tests/data/precalib/precalib_base_data/calib.left.distortion.txt')
right_intrinsics = np.loadtxt('tests/data/precalib/precalib_base_data/calib.right.intrinsics.txt')
right_distortion = np.loadtxt('tests/data/precalib/precalib_base_data/calib.right.distortion.txt')
l2r = np.loadtxt('tests/data/precalib/precalib_base_data/calib.l2r.txt')
l2r_rmat = l2r[0:3, 0:3]
l2r_tvec = l2r[0:3, 3]
calib_dir = 'tests/data/precalib/data_moved_scope'
calib_driver = get_calib_driver(calib_dir)
stereo_reproj_err, stereo_recon_err, _ = \
calib_driver.calibrate(
override_left_intrinsics=left_intrinsics,
override_left_distortion=left_distortion,
override_right_intrinsics=right_intrinsics,
override_right_distortion=right_distortion,
override_l2r_rmat=l2r_rmat,
override_l2r_tvec=l2r_tvec)
tracked_reproj_err, tracked_recon_err, _ = \
calib_driver.handeye_calibration()
print(stereo_reproj_err, stereo_recon_err, tracked_reproj_err, tracked_recon_err)
> assert stereo_reproj_err < 4.5
E assert 34.74794862170866 < 4.5
tests/video/test_precalib.py:124: AssertionError
================================================================================================ warnings summary =================================================================================================
tests/video/test_data_param_io.py: 18 warnings
tests/video/test_iterative.py: 10 warnings
tests/video/test_precalib.py: 18 warnings
tests/video/test_video_calibration.py: 9 warnings
/home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/sksurgerycalibration/video/video_calibration_utils.py:106: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
np.zeros((len(ids), 1), dtype=np.int)
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
============================================================================================= short test summary info =============================================================================================
FAILED tests/video/test_precalib.py::test_charuco_dataset_B - assert 7.116922375480872 < 1
FAILED tests/video/test_precalib.py::test_precalbration - assert 34.74794862170866 < 4.5
=================================================================================== 2 failed, 23 passed, 55 warnings in 40.91s ====================================================================================
ERROR: InvocationError for command /home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/.tox/py37/bin/coverage run -a --source ./sksurgerycalibration -m pytest -v -s ./tests/ (exited with code 1)
_____________________________________________________________________________________________________ summary _____________________________________________________________________________________________________
ERROR: py37: commands failed
Testing test_precalib.py with local conda env using - opencv-contrib-python-headless==4.5.3.56
Logs:
==================================================================================================== FAILURES =====================================================================================================
_____________________________________________________________________________________________ test_charuco_dataset_B ______________________________________________________________________________________________
def test_charuco_dataset_B():
calib_dir = 'tests/data/precalib/data_moved_scope'
calib_driver = get_calib_driver(calib_dir)
stereo_reproj_err, stereo_recon_err, _ = \
calib_driver.calibrate()
tracked_reproj_err, tracked_recon_err, _ = \
calib_driver.handeye_calibration()
print(stereo_reproj_err, stereo_recon_err, tracked_reproj_err, tracked_recon_err)
> assert stereo_reproj_err < 1
E assert 7.116922375480872 < 1
tests/video/test_precalib.py:92: AssertionError
_______________________________________________________________________________________________ test_precalbration ________________________________________________________________________________________________
def test_precalbration():
""" Use intrinsics from A to calibration B, currently failing. """
left_intrinsics = np.loadtxt('tests/data/precalib/precalib_base_data/calib.left.intrinsics.txt')
left_distortion = np.loadtxt('tests/data/precalib/precalib_base_data/calib.left.distortion.txt')
right_intrinsics = np.loadtxt('tests/data/precalib/precalib_base_data/calib.right.intrinsics.txt')
right_distortion = np.loadtxt('tests/data/precalib/precalib_base_data/calib.right.distortion.txt')
l2r = np.loadtxt('tests/data/precalib/precalib_base_data/calib.l2r.txt')
l2r_rmat = l2r[0:3, 0:3]
l2r_tvec = l2r[0:3, 3]
calib_dir = 'tests/data/precalib/data_moved_scope'
calib_driver = get_calib_driver(calib_dir)
stereo_reproj_err, stereo_recon_err, _ = \
calib_driver.calibrate(
override_left_intrinsics=left_intrinsics,
override_left_distortion=left_distortion,
override_right_intrinsics=right_intrinsics,
override_right_distortion=right_distortion,
override_l2r_rmat=l2r_rmat,
override_l2r_tvec=l2r_tvec)
tracked_reproj_err, tracked_recon_err, _ = \
calib_driver.handeye_calibration()
print(stereo_reproj_err, stereo_recon_err, tracked_reproj_err, tracked_recon_err)
> assert stereo_reproj_err < 4.5
E assert 34.74794862170866 < 4.5
tests/video/test_precalib.py:124: AssertionError
================================================================================================ warnings summary =================================================================================================
tests/video/test_data_param_io.py: 18 warnings
tests/video/test_iterative.py: 10 warnings
tests/video/test_precalib.py: 18 warnings
tests/video/test_video_calibration.py: 9 warnings
/home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/sksurgerycalibration/video/video_calibration_utils.py:106: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
np.zeros((len(ids), 1), dtype=np.int)
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
============================================================================================= short test summary info =============================================================================================
FAILED tests/video/test_precalib.py::test_charuco_dataset_B - assert 7.116922375480872 < 1
FAILED tests/video/test_precalib.py::test_precalbration - assert 34.74794862170866 < 4.5
=================================================================================== 2 failed, 23 passed, 55 warnings in 41.34s ====================================================================================
ERROR: InvocationError for command /home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/.tox/py37/bin/coverage run -a --source ./sksurgerycalibration -m pytest -v -s ./tests/ (exited with code 1)
_____________________________________________________________________________________________________ summary _____________________________________________________________________________________________________
ERROR: py37: commands failed
Let's consider previous unit tests https://github.com/SciKit-Surgery/scikit-surgerycalibration/actions/runs/2022284109 :
to create branch from such commit
git checkout -b 48-clean-unit-test 38bc2e69333e417bc45d4c94b725587d3b9c145b
Result shows errors : https://github.com/SciKit-Surgery/scikit-surgerycalibration/actions/runs/3684161051/jobs/6233563873
Steve and I discussed other potential reasons for errors in the unit test that can be related to other package versions. Maybe start trying opencv-contrib-python-headless==4.2 also look #35 that made use of opencv contrib <=4.5.1. Look to other versions in the pype! https://pypi.org/project/opencv-contrib-python-headless/#history
Commit b8196bc seems to fix the unit test failures (i.e. opencv < 4.6). Actions still shows failure because of linting. pylintrc needs updating.
Indeed <4.6 fix unit test failures. Just checked that my current local conda env with python3.7 has opencv-contrib-python-headless 4.5.5.64 pypi_0 pypi, (also matching the unit test vesion opencv-contrib-python-headless-4.5.5.64) but got this for local log tox:
========================================================================== FAILURES ==========================================================================
____________________________________________________________________ test_stereo_davinci _____________________________________________________________________
def test_stereo_davinci():
left_images = []
files = glob.glob('tests/data/ChAruco_LR_frames_Steve_Axis_Tests/ExtractedFrames_L/*.jpg')
files.sort()
for file in files:
image = cv2.imread(file)
print("Loaded:" + str(file))
left_images.append(image)
assert(len(left_images) == 59)
right_images = []
files = glob.glob('tests/data/ChAruco_LR_frames_Steve_Axis_Tests/ExtractedFrames_R/*.jpg')
files.sort()
for file in files:
image = cv2.imread(file)
print("Loaded:" + str(file))
right_images.append(image)
assert (len(right_images) == 59)
ref_img = cv2.imread('tests/data/2020_01_20_storz/pattern_4x4_19x26_5_4_with_inset_9x14.png')
minimum_number_of_points_per_image = 50
detector = pd.CharucoPlusChessboardPointDetector(ref_img,
error_if_no_chessboard=False) # Try to accept as many as possible.
calibrator = sc.StereoVideoCalibrationDriver(detector, detector, minimum_number_of_points_per_image)
for i, _ in enumerate(left_images):
try:
number_left, number_right = calibrator.grab_data(left_images[i], right_images[i])
if number_left < minimum_number_of_points_per_image:
print("Image pair:" + str(i) + ", left image, SKIPPED, due to not enough points")
if number_right < minimum_number_of_points_per_image:
print("Image pair:" + str(i) + ", right image, SKIPPED, due to not enough points")
except ValueError as ve:
print("Image pair:" + str(i) + ", FAILED, due to:" + str(ve))
except TypeError as te:
print("Image pair:" + str(i) + ", FAILED, due to:" + str(te))
reproj_err, recon_err, params = calibrator.calibrate()
print("Reproj:" + str(reproj_err))
print("Recon:" + str(recon_err))
> assert reproj_err < 1.1
E assert 82521.00771341282 < 1.1
tests/video/test_charuco_plus_chessboard.py:55: AssertionError
_______________________________________________________________ test_handeye_calibration_mono ________________________________________________________________
def test_handeye_calibration_mono():
"""
Load mono data (only using left channel) and tracking, do video and
handeye calibration, compare results against expected values.
"""
images = load_images_from_glob(
'tests/data/2020_01_20_storz/12_50_30/calib.left.*.png')
device_tracking = load_tracking_from_glob(
'tests/data/2020_01_20_storz/12_50_30/calib.device_tracking.*.txt')
obj_tracking = load_tracking_from_glob(
'tests/data/2020_01_20_storz/12_50_30/calib.calib_obj_tracking.*.txt')
ref_img = cv2.imread(
'tests/data/2020_01_20_storz/pattern_4x4_19x26_5_4_with_inset_9x14.png')
assert len(images) == 10
assert len(device_tracking) == 10
assert len(obj_tracking) == 10
min_number_of_points_per_image = 50
detector = \
chpd.CharucoPlusChessboardPointDetector(ref_img,
error_if_no_chessboard=False)
calibrator = \
mcd.MonoVideoCalibrationDriver(detector, min_number_of_points_per_image)
# Grab data from images/tracking arrays
for image, device, calib_obj in zip(images, device_tracking, obj_tracking):
successful = calibrator.grab_data(image, device, calib_obj)
assert successful > 0
reproj_err_1, _ = calibrator.calibrate()
> assert reproj_err_1 == pytest.approx(1., rel=0.2)
E assert 146.10223709746785 == 1.0 ± 2.0e-01
E comparison failed
E Obtained: 146.10223709746785
E Expected: 1.0 ± 2.0e-01
tests/video/test_hand_eye.py:78: AssertionError
______________________________________________________________ test_handeye_calibration_stereo _______________________________________________________________
def test_handeye_calibration_stereo():
"""
Load Stereo data and tracking, do video and
handeye calibration, compare results against expected values.
"""
left_images = load_images_from_glob(
'tests/data/2020_01_20_storz/12_50_30/calib.left.*.png')
right_images = load_images_from_glob(
'tests/data/2020_01_20_storz/12_50_30/calib.right.*.png')
device_tracking = load_tracking_from_glob(
'tests/data/2020_01_20_storz/12_50_30/calib.device_tracking.*.txt')
obj_tracking = load_tracking_from_glob(
'tests/data/2020_01_20_storz/12_50_30/calib.calib_obj_tracking.*.txt')
ref_img = cv2.imread(
'tests/data/2020_01_20_storz/pattern_4x4_19x26_5_4_with_inset_9x14.png')
assert len(left_images) == 10
assert len(right_images) == 10
assert len(device_tracking) == 10
assert len(obj_tracking) == 10
min_number_of_points_per_image = 50
detector = \
chpd.CharucoPlusChessboardPointDetector(ref_img,
charuco_filtering=True,
error_if_no_chessboard=False)
calibrator = \
scd.StereoVideoCalibrationDriver(
detector, detector, min_number_of_points_per_image)
# Grab data from images/tracking arrays
for left, right, device, calib_obj in \
zip(left_images, right_images, device_tracking, obj_tracking):
num_left, num_right = calibrator.grab_data(left,
right,
device,
calib_obj)
assert num_left > 0
assert num_right > 0
assert num_left < 480
assert num_right < 480
reproj_err_1, recon_err_1, _ = calibrator.calibrate()
> assert reproj_err_1 == pytest.approx(0.6, rel=0.2)
E assert 23788027320.1106 == 0.6 ± 1.2e-01
E comparison failed
E Obtained: 23788027320.1106
E Expected: 0.6 ± 1.2e-01
tests/video/test_hand_eye.py:140: AssertionError
___________________________________________________________________ test_charuco_dataset_B ___________________________________________________________________
def test_charuco_dataset_B():
calib_dir = 'tests/data/precalib/data_moved_scope'
calib_driver = get_calib_driver(calib_dir)
stereo_reproj_err, stereo_recon_err, _ = \
calib_driver.calibrate()
tracked_reproj_err, tracked_recon_err, _ = \
calib_driver.handeye_calibration()
print(stereo_reproj_err, stereo_recon_err, tracked_reproj_err, tracked_recon_err)
> assert stereo_reproj_err < 1
E assert 7.116922375480872 < 1
tests/video/test_precalib.py:92: AssertionError
_____________________________________________________________________ test_precalbration _____________________________________________________________________
def test_precalbration():
""" Use intrinsics from A to calibration B, currently failing. """
left_intrinsics = np.loadtxt('tests/data/precalib/precalib_base_data/calib.left.intrinsics.txt')
left_distortion = np.loadtxt('tests/data/precalib/precalib_base_data/calib.left.distortion.txt')
right_intrinsics = np.loadtxt('tests/data/precalib/precalib_base_data/calib.right.intrinsics.txt')
right_distortion = np.loadtxt('tests/data/precalib/precalib_base_data/calib.right.distortion.txt')
l2r = np.loadtxt('tests/data/precalib/precalib_base_data/calib.l2r.txt')
l2r_rmat = l2r[0:3, 0:3]
l2r_tvec = l2r[0:3, 3]
calib_dir = 'tests/data/precalib/data_moved_scope'
calib_driver = get_calib_driver(calib_dir)
stereo_reproj_err, stereo_recon_err, _ = \
calib_driver.calibrate(
override_left_intrinsics=left_intrinsics,
override_left_distortion=left_distortion,
override_right_intrinsics=right_intrinsics,
override_right_distortion=right_distortion,
override_l2r_rmat=l2r_rmat,
override_l2r_tvec=l2r_tvec)
tracked_reproj_err, tracked_recon_err, _ = \
calib_driver.handeye_calibration()
print(stereo_reproj_err, stereo_recon_err, tracked_reproj_err, tracked_recon_err)
> assert stereo_reproj_err < 4.5
E assert 34.74794862170866 < 4.5
tests/video/test_precalib.py:124: AssertionError
====================================================================== warnings summary ======================================================================
tests/video/test_charuco_plus_chessboard.py: 52 warnings
tests/video/test_chessboard_calibration.py: 63 warnings
tests/video/test_data_param_io.py: 18 warnings
tests/video/test_hand_eye.py: 10 warnings
tests/video/test_handeye_dots.py: 406 warnings
tests/video/test_iterative.py: 10 warnings
tests/video/test_precalib.py: 18 warnings
tests/video/test_video_calibration.py: 9 warnings
/home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/sksurgerycalibration/video/video_calibration_utils.py:106: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
np.zeros((len(ids), 1), dtype=np.int)
tests/video/test_handeye_dots.py::test_gx_vs_cv
tests/video/test_handeye_dots.py::test_fixed_p2m
tests/video/test_handeye_dots.py::test_tracked_vs_stationary
tests/video/test_handeye_dots.py::tests_for_smart_liver
/home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/sksurgerycalibration/video/video_calibration_metrics.py:247: RuntimeWarning: overflow encountered in square
sse = sse + np.sum(np.square(diff))
tests/video/test_handeye_dots.py::test_gx_vs_cv
tests/video/test_handeye_dots.py::test_fixed_p2m
tests/video/test_handeye_dots.py::test_tracked_vs_stationary
tests/video/test_handeye_dots.py::tests_for_smart_liver
/home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/.tox/py37/lib/python3.7/site-packages/numpy/core/fromnumeric.py:86: RuntimeWarning: overflow encountered in reduce
return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
tests/video/test_handeye_dots.py::test_gx_vs_cv
tests/video/test_handeye_dots.py::test_fixed_p2m
tests/video/test_handeye_dots.py::test_tracked_vs_stationary
tests/video/test_handeye_dots.py::tests_for_smart_liver
/home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/.tox/py37/lib/python3.7/site-packages/scipy/optimize/optimize.py:2215: RuntimeWarning: invalid value encountered in double_scalars
tmp2 = (x - v) * (fx - fw)
tests/video/test_handeye_dots.py::test_gx_vs_cv
tests/video/test_handeye_dots.py::test_fixed_p2m
/home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/sksurgerycalibration/video/video_calibration_metrics.py:95: RuntimeWarning: overflow encountered in square
rse = rse + np.sum(np.square(diff_right))
tests/video/test_handeye_dots.py::test_gx_vs_cv
tests/video/test_handeye_dots.py::test_fixed_p2m
/home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/sksurgerycalibration/video/video_calibration_metrics.py:94: RuntimeWarning: overflow encountered in square
lse = lse + np.sum(np.square(diff_left))
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
================================================================== short test summary info ===================================================================
FAILED tests/video/test_charuco_plus_chessboard.py::test_stereo_davinci - assert 82521.00771341282 < 1.1
FAILED tests/video/test_hand_eye.py::test_handeye_calibration_mono - assert 146.10223709746785 == 1.0 ± 2.0e-01
FAILED tests/video/test_hand_eye.py::test_handeye_calibration_stereo - assert 23788027320.1106 == 0.6 ± 1.2e-01
FAILED tests/video/test_precalib.py::test_charuco_dataset_B - assert 7.116922375480872 < 1
FAILED tests/video/test_precalib.py::test_precalbration - assert 34.74794862170866 < 4.5
=================================================== 5 failed, 30 passed, 602 warnings in 287.77s (0:04:47) ===================================================
ERROR: InvocationError for command /home/mxochicale/repositories/Scikit-Surgery/scikit-surgerycalibration/.tox/py37/bin/coverage run -a --source ./sksurgerycalibration -m pytest -v -s ./tests/ (exited with code 1)
Would python-version: 3.7 be a good python version for deploy?; 3.7 and 3.8 for unit tests; and 3.9, 3.10 as experimental versions ?
I think so, that's what most of our other libraries are moving towards.