Example Fisheye To Equirectangular

Equirectangular images are a common way to encode a full 360 view. They are typically generated by combining multiple images together. This example demonstrates how to combine two fisheye camera view from the 360 camera into a single equirectangular view.

Example Code

/** * Demonstrates how to combine multiple images together into a single view. A 360 camera was used to generate * the two input fisheye images. Each camera has been calibrated independently and the extrinsics between the two * cameras is assume to be known. Because of how the fisheye image is modeled a mask is required to label pixels * outside the FOV that should not be considered. * * @author Peter Abeles */publicclassExampleFisheyeToEquirectangular{/** * Creates a mask telling the algorithm which pixels are valid and which are not. The field-of-view (FOV) of the * camera is known so we will use that information to do a better job of filtering out invalid pixels than * it can do alone. */publicstaticGrayU8createMask(CameraUniversalOmnimodel,LensDistortionWideFOVdistortion,doublefov){GrayU8mask=newGrayU8(model.width,model.height);Point2Transform3_F64p2s=distortion.undistortPtoS_F64();Point3D_F64ref=newPoint3D_F64(0,0,1);Point3D_F64X=newPoint3D_F64();p2s.compute(model.cx,model.cy,X);for(inty=0;y<model.height;y++){for(intx=0;x<model.width;x++){p2s.compute(x,y,X);if(Double.isNaN(X.x)||Double.isNaN(X.y)||Double.isNaN(X.z)){continue;}doubleangle=UtilVector3D_F64.acute(ref,X);if(Double.isNaN(angle)){continue;}if(angle<=fov/2.0)mask.unsafe_set(x,y,1);}}returnmask;}publicstaticvoidmain(String[]args){// Path to image data and calibration dataStringfisheyePath=UtilIO.pathExample("fisheye/theta");// load the fisheye camera parametersCameraUniversalOmnimodel0=CalibrationIO.load(newFile(fisheyePath,"front.yaml"));CameraUniversalOmnimodel1=CalibrationIO.load(newFile(fisheyePath,"back.yaml"));LensDistortionWideFOVdistort0=newLensDistortionUniversalOmni(model0);LensDistortionWideFOVdistort1=newLensDistortionUniversalOmni(model1);ImageType<Planar<GrayF32>>imageType=ImageType.pl(3,GrayF32.class);InterpolatePixel<Planar<GrayF32>>interp=FactoryInterpolation.createPixel(0,255,InterpolationType.BILINEAR,BorderType.ZERO,imageType);ImageDistort<Planar<GrayF32>,Planar<GrayF32>>distort=FactoryDistort.distort(false,interp,imageType);//This will create an equirectangular image with 800 x 400 pixelsMultiCameraToEquirectangular<Planar<GrayF32>>alg=newMultiCameraToEquirectangular<>(distort,800,400,imageType);// this is an important parameter and is used to filter out falsely mirrored pixelsalg.setMaskToleranceAngle(UtilAngle.radian(0.1f));GrayU8mask0=createMask(model0,distort0,UtilAngle.radian(182));// camera has a known FOV of 185 degreesGrayU8mask1=createMask(model1,distort1,UtilAngle.radian(182));// the edges are likely to be noisy,// so crop it a bit..// Rotate camera axis so that +x is forward and not +z and make it visually pleasingFMatrixRMajadjR=ConvertRotation3D_F32.eulerToMatrix(EulerType.XYZ,GrlConstants.F_PI/2,0,0,null);// Rotation from the front camera to the back facing camera.// This is only an approximation. Should be determined through calibration.FMatrixRMajf2b=ConvertRotation3D_F32.eulerToMatrix(EulerType.ZYX,GrlConstants.F_PI,0,0,null);Se3_F32frontToFront=newSe3_F32();frontToFront.setRotation(adjR);Se3_F32frontToBack=newSe3_F32();CommonOps_FDRM.mult(f2b,adjR,frontToBack.R);// add the camera and specify which pixels are valid. These functions precompute the entire transform// and can be relatively slow, but generating the equirectangular image should be much fasteralg.addCamera(frontToBack,distort0,mask0);alg.addCamera(frontToFront,distort1,mask1);// Load fisheye RGB imageBufferedImagebuffered0=UtilImageIO.loadImage(fisheyePath,"front_table.jpg");Planar<GrayF32>fisheye0=ConvertBufferedImage.convertFrom(buffered0,true,ImageType.pl(3,GrayF32.class));BufferedImagebuffered1=UtilImageIO.loadImage(fisheyePath,"back_table.jpg");Planar<GrayF32>fisheye1=ConvertBufferedImage.convertFrom(buffered1,true,ImageType.pl(3,GrayF32.class));List<Planar<GrayF32>>images=newArrayList<>();images.add(fisheye0);images.add(fisheye1);alg.render(images);BufferedImageequiOut=ConvertBufferedImage.convertTo(alg.getRenderedImage(),null,true);ShowImages.showWindow(equiOut,"Dual Fisheye to Equirectangular",true);}}