Fish eye undistortion doesnt work

[mbaranovskyi]

Unity 2020.3 lts

When I am trying to undistort fish eye image with Calib3d.fisheye_undistortImage I am getting fully black texture. At the same time if I just change API call to Calib3d.undistort I can get the normal texture just incorrect undistortion result cause of camera model. No logs, no documentation, no workarounds found.

here is code example.
`
[RequireComponent(typeof(RawImage))]
public class UndistortImage : MonoBehaviour
{
private RawImage m_image;

private float[] distCoeffs = new float[]
{
    -0.6191183790002164f,
    0.4831049727953358f,
    0.0f,
    0.0f,
    0.10673271388949125f,
    -0.2457000633328616f,
    0.17837574249218238f,
    0.34149105792940876f,
    0.0f,
    0.0f,
    0.0f,
    0.0f,
    0.0f,
    0.0f
};

void Start()
{
    m_image = GetComponent<RawImage>();

    Mat src = new Mat(m_image.texture.height, m_image.texture.width, CvType.CV_8UC3);
    Mat dst = new Mat(m_image.texture.height, m_image.texture.width, CvType.CV_8UC3);

    Utils.texture2DToMat((Texture2D) m_image.texture, src);

    // do you stuff here
    Calib3d.fisheye_undistortImage(src,
                                    dst,
                                    CreateCameraMatrix(m_image.texture.width, m_image.texture.height),
                                    new MatOfFloat(distCoeffs));
    
    // Calib3d.undistort(src,
    //                                dst,
    //                                CreateCameraMatrix(m_image.texture.width, m_image.texture.height),
    //                                new MatOfFloat(distCoeffs));
    
    var texture = new Texture2D(
                                m_image.texture.width,
                                m_image.texture.height,
                                TextureFormat.RGB24,
                                m_image.texture.mipmapCount,
                                true
                               );
    
    Utils.matToTexture2D(dst, texture);
    m_image.texture = texture;
}

private Mat CreateCameraMatrix(float width, float height)
{
    int max_d = (int) Mathf.Max(width, height);
    double fx = max_d;
    double fy = max_d;
    double cx = width / 2.0f;
    double cy = height / 2.0f;

    Mat camMatrix = new Mat(3, 3, CvType.CV_64FC1);
    camMatrix.put(0, 0, fx);
    camMatrix.put(0, 1, 0);
    camMatrix.put(0, 2, cx);
    camMatrix.put(1, 0, 0);
    camMatrix.put(1, 1, fy);
    camMatrix.put(1, 2, cy);
    camMatrix.put(2, 0, 0);
    camMatrix.put(2, 1, 0);
    camMatrix.put(2, 2, 1.0f);

    return camMatrix;
}

}
`

help, please

[Enox Software]

We have created a simple example of "Calib3d.fisheye_undistortImage()" functon.
Try it out.
FisheyeUndistortImageExample.zip

using OpenCVForUnity.Calib3dModule;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.UnityUtils;
using UnityEngine;
using UnityEngine.UI;

// This example was created based on code posted in the following forum topic.
// https://stackoverflow.com/questions/64903037/undistort-a-fisheye-image-using-cvfisheyeundistortimage
// 
// https://docs.opencv.org/4.1.2/db/d58/group__calib3d__fisheye.html#ga167df4b00a6fd55287ba829fbf9913b9
// https://stackoverflow.com/questions/34316306/opencv-fisheye-calibration-cuts-too-much-of-the-resulting-image
// https://github.com/Kazuhito00/OpenCV-CameraCalibration-Example/blob/main/02-02_fisheyeUndistort.py

[RequireComponent(typeof(RawImage))]
public class FisheyeUndistortImageExample : MonoBehaviour
{
    private RawImage m_image;

    private double[] cameraMatrix_arr = new double[]
    {
        286.7037963867188,
        0.0,
        413.3463134765625,
        0.0,
        286.7817993164062,
        397.1785888671875,
        0.0,
        0.0,
        1.0
    };

    private double[] distortionCoeffs_arr = new double[]
    {
        -0.01078350003808737,
        0.04842806980013847,
        -0.04542399942874908,
        0.008737384341657162
    };

    void Start()
    {
        // Outputs native code exceptions.
        Utils.setDebugMode(true);
        //

        m_image = GetComponent<RawImage>();

        Mat input_frame = new Mat(m_image.texture.height, m_image.texture.width, CvType.CV_8UC3);
        Mat undistort = new Mat(m_image.texture.height, m_image.texture.width, CvType.CV_8UC3);

        Utils.texture2DToMat((Texture2D)m_image.texture, input_frame);


        Mat cameraMatrix = new Mat(3, 3, CvType.CV_64FC1);
        cameraMatrix.put(0, 0, cameraMatrix_arr);

        Mat distortionCoeffs = new Mat(4, 1, CvType.CV_64FC1);
        distortionCoeffs.put(0, 0, distortionCoeffs_arr);

        Size size = new Size(input_frame.width(), input_frame.height());


        // simply way
        Calib3d.fisheye_undistortImage(input_frame, undistort, cameraMatrix, distortionCoeffs, cameraMatrix, size);

        /*
        // 2 function combination way

        Mat E = Mat.eye(3, 3, CvType.CV_64FC1);
        Mat map1 = new Mat();
        Mat map2 = new Mat();
        Calib3d.fisheye_initUndistortRectifyMap(cameraMatrix, distortionCoeffs, E, cameraMatrix, size, CvType.CV_16SC2, map1, map2);
        Imgproc.remap(input_frame, undistort, map1, map2, Imgproc.INTER_LINEAR, Core.BORDER_CONSTANT);
        */


        var texture = new Texture2D(m_image.texture.width, m_image.texture.height, TextureFormat.RGB24, false);

        Utils.matToTexture2D(undistort, texture);

        m_image.texture = texture;

        //
        Utils.setDebugMode(false);
        //
    }
}

[mbaranovskyi]

Thank you, it works!
2 more questions regarding to this topic:

1. is there a way to get full undistorted image? couse for now undistortion removing like 60% of image
   this is the result of this one
   

2. is it possible to make fisheye_distort call? in original OpenCV they have fisheye_distortPoints, but in OpenCVForUnity i cant see such method

[Enox Software]

1. is there a way to get full undistorted image? couse for now undistortion removing like 60% of image

The explanations on this page will be very helpful.
https://medium.com/@kennethjiang/calibrate-fisheye-lens-using-opencv-part-2-13990f1b157f

        Mat E = Mat.eye(3, 3, CvType.CV_64FC1);
        Mat map1 = new Mat();
        Mat map2 = new Mat();
        Mat new_K = new Mat();
        Mat scaled_K = cameraMatrix.clone();
        Calib3d.fisheye_estimateNewCameraMatrixForUndistortRectify(cameraMatrix, distortionCoeffs, size, E, new_K, 1.0);
        Calib3d.fisheye_initUndistortRectifyMap(scaled_K, distortionCoeffs, E, new_K, size, CvType.CV_16SC2, map1, map2);
        Imgproc.remap(input_frame, undistort, map1, map2, Imgproc.INTER_LINEAR, Core.BORDER_CONSTANT);

https://stackoverflow.com/a/44009548
https://stackoverflow.com/a/60521629

2. is it possible to make fisheye_distort call? in original OpenCV they have fisheye_distortPoints, but in OpenCVForUnity i cant see such method

"Calib3d.fisheye_distortPoints();" does indeed exist.
http://enoxsoftware.github.io/OpenCVForUnity/3.0.0/doc/html/class_open_c_v_for_unity_1_1_calib3d_module_1_1_calib3d.html#a71f103e8a431ef425ea687f421ad5397