diff options
Diffstat (limited to 'jni/feature_stab/src/dbreg/dbreg.h')
| -rw-r--r-- | jni/feature_stab/src/dbreg/dbreg.h | 581 |
1 files changed, 581 insertions, 0 deletions
diff --git a/jni/feature_stab/src/dbreg/dbreg.h b/jni/feature_stab/src/dbreg/dbreg.h new file mode 100644 index 000000000..4eb244481 --- /dev/null +++ b/jni/feature_stab/src/dbreg/dbreg.h @@ -0,0 +1,581 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#pragma once + +#ifdef _WIN32 +#ifdef DBREG_EXPORTS +#define DBREG_API __declspec(dllexport) +#else +#define DBREG_API __declspec(dllimport) +#endif +#else +#define DBREG_API +#endif + +// @jke - the next few lines are for extracting timing data. TODO: Remove after test +#define PROFILE 0 + +#include "dbstabsmooth.h" + +#include <db_feature_detection.h> +#include <db_feature_matching.h> +#include <db_rob_image_homography.h> + +#if PROFILE + #include <sys/time.h> +#endif + +/*! \mainpage db_FrameToReferenceRegistration + + \section intro Introduction + + db_FrameToReferenceRegistration provides a simple interface to a set of sophisticated algorithms for stabilizing + video sequences. As its name suggests, the class is used to compute parameters that will allow us to warp incoming video + frames and register them with respect to a so-called <i>reference</i> frame. The reference frame is simply the first + frame of a sequence; the registration process is that of estimating the parameters of a warp that can be applied to + subsequent frames to make those frames align with the reference. A video made up of these warped frames will be more + stable than the input video. + + For more technical information on the internal structure of the algorithms used within the db_FrameToRegistration class, + please follow this <a href="../Sarnoff image registration.docx">link</a>. + + \section usage Usage + In addition to the class constructor, there are two main functions of db_FrameToReferenceRegistration that are of + interest to the programmer. db_FrameToReferenceRegistration::Init(...) is used to initialize the parameters of the + registration algorithm. db_FrameToReferenceRegistration::AddFrame(...) is the method by which each new video frame + is introduced to the registration algorithm, and produces the estimated registration warp parameters. + + The following example illustrates how the major methods of the class db_FrameToReferenceRegistration can be used together + to calculate the registration parameters for an image sequence. In the example, the calls to the methods of + db_FrameToReferenceRegistration match those found in the API, but supporting code should be considered pseudo-code. + For a more complete example, please consult the source code for dbregtest. + + + \code + // feature-based image registration class: + db_FrameToReferenceRegistration reg; + + // Image data + const unsigned char * const * image_storage; + + // The 3x3 frame to reference registration parameters + double frame_to_ref_homography[9]; + + // a counter to count the number of frames processed. + unsigned long frame_counter; + // ... + + // main loop - keep going while there are images to process. + while (ImagesAreAvailable) + { + // Call functions to place latest data into image_storage + // ... + + // if the registration object is not yet initialized, then do so + // The arguments to this function are explained in the accompanying + // html API documentation + if (!reg.Initialized()) + { + reg.Init(w,h,motion_model_type,25,linear_polish,quarter_resolution, + DB_POINT_STANDARDDEV,reference_update_period, + do_motion_smoothing,motion_smoothing_gain, + DB_DEFAULT_NR_SAMPLES,DB_DEFAULT_CHUNK_SIZE, + nr_corners,max_disparity); + } + + // Present the new image data to the registration algorithm, + // with the result being stored in the frame_to_ref_homography + // variable. + reg.AddFrame(image_storage,frame_to_ref_homography); + + // frame_to_ref_homography now contains the stabilizing transform + // use this to warp the latest image for display, etc. + + // if this is the first frame, we need to tell the registration + // class to store the image as its reference. Otherwise, AddFrame + // takes care of that. + if (frame_counter == 0) + { + reg.UpdateReference(image_storage); + } + + // increment the frame counter + frame_counter++; + } + + \endcode + + */ + +/*! + * Performs feature-based frame to reference image registration. + */ +class DBREG_API db_FrameToReferenceRegistration +{ +public: + db_FrameToReferenceRegistration(void); + ~db_FrameToReferenceRegistration(); + + /*! + * Set parameters and allocate memory. Note: The default values of these parameters have been set to the values used for the android implementation (i.e. the demo APK). + * \param width image width + * \param height image height + * \param homography_type see definitions in \ref LMRobImageHomography + * \param max_iterations max number of polishing steps + * \param linear_polish whether to perform a linear polishing step after RANSAC + * \param quarter_resolution whether to process input images at quarter resolution (for computational efficiency) + * \param scale Cauchy scale coefficient (see db_ExpCauchyReprojectionError() ) + * \param reference_update_period how often to update the alignment reference (in units of number of frames) + * \param do_motion_smoothing whether to perform display reference smoothing + * \param motion_smoothing_gain weight factor to reflect how fast the display reference must follow the current frame if motion smoothing is enabled + * \param nr_samples number of times to compute a hypothesis + * \param chunk_size size of cost chunks + * \param cd_target_nr_corners target number of corners for corner detector + * \param cm_max_disparity maximum disparity search range for corner matcher (in units of ratio of image width) + * \param cm_use_smaller_matching_window if set to true, uses a correlation window of 5x5 instead of the default 11x11 + * \param cd_nr_horz_blocks the number of horizontal blocks for the corner detector to partition the image + * \param cd_nr_vert_blocks the number of vertical blocks for the corner detector to partition the image + */ + void Init(int width, int height, + int homography_type = DB_HOMOGRAPHY_TYPE_DEFAULT, + int max_iterations = DB_DEFAULT_MAX_ITERATIONS, + bool linear_polish = false, + bool quarter_resolution = true, + double scale = DB_POINT_STANDARDDEV, + unsigned int reference_update_period = 3, + bool do_motion_smoothing = false, + double motion_smoothing_gain = 0.75, + int nr_samples = DB_DEFAULT_NR_SAMPLES, + int chunk_size = DB_DEFAULT_CHUNK_SIZE, + int cd_target_nr_corners = 500, + double cm_max_disparity = 0.2, + bool cm_use_smaller_matching_window = false, + int cd_nr_horz_blocks = 5, + int cd_nr_vert_blocks = 5); + + /*! + * Reset the transformation type that is being use to perform alignment. Use this to change the alignment type at run time. + * \param homography_type the type of transformation to use for performing alignment (see definitions in \ref LMRobImageHomography) + */ + void ResetHomographyType(int homography_type) { m_homography_type = homography_type; } + + /*! + * Enable/Disable motion smoothing. Use this to turn motion smoothing on/off at run time. + * \param enable flag indicating whether to turn the motion smoothing on or off. + */ + void ResetSmoothing(bool enable) { m_do_motion_smoothing = enable; } + + /*! + * Align an inspection image to an existing reference image, update the reference image if due and perform motion smoothing if enabled. + * \param im new inspection image + * \param H computed transformation from reference to inspection coordinate frame. Identity is returned if no reference frame was set. + * \param force_reference make this the new reference image + */ + int AddFrame(const unsigned char * const * im, double H[9], bool force_reference=false, bool prewarp=false); + + /*! + * Returns true if Init() was run. + */ + bool Initialized() const { return m_initialized; } + + /*! + * Returns true if the current frame is being used as the alignment reference. + */ + bool IsCurrentReference() const { return m_current_is_reference; } + + /*! + * Returns true if we need to call UpdateReference now. + */ + bool NeedReferenceUpdate(); + + /*! + * Returns the pointer reference to the alignment reference image data + */ + unsigned char ** GetReferenceImage() { return m_reference_image; } + + /*! + * Returns the pointer reference to the double array containing the homogeneous coordinates for the matched reference image corners. + */ + double * GetRefCorners() { return m_corners_ref; } + /*! + * Returns the pointer reference to the double array containing the homogeneous coordinates for the matched inspection image corners. + */ + double * GetInsCorners() { return m_corners_ins; } + /*! + * Returns the number of correspondences between the reference and inspection images. + */ + int GetNrMatches() { return m_nr_matches; } + + /*! + * Returns the number of corners detected in the current reference image. + */ + int GetNrRefCorners() { return m_nr_corners_ref; } + + /*! + * Returns the pointer to an array of indices that were found to be RANSAC inliers from the matched corner lists. + */ + int* GetInliers() { return m_inlier_indices; } + + /*! + * Returns the number of inliers from the RANSAC matching step. + */ + int GetNrInliers() { return m_num_inlier_indices; } + + //std::vector<int>& GetInliers(); + //void Polish(std::vector<int> &inlier_indices); + + /*! + * Perform a linear polishing step by re-estimating the alignment transformation using the RANSAC inliers. + * \param inlier_indices pointer to an array of indices that were found to be RANSAC inliers from the matched corner lists. + * \param num_inlier_indices number of inliers i.e. the length of the array passed as the first argument. + */ + void Polish(int *inlier_indices, int &num_inlier_indices); + + /*! + * Reset the motion smoothing parameters to their initial values. + */ + void ResetMotionSmoothingParameters() { m_stab_smoother.Init(); } + + /*! + * Update the alignment reference image to the specified image. + * \param im pointer to the image data to be used as the new alignment reference. + * \param subsample boolean flag to control whether the function should internally subsample the provided image to the size provided in the Init() function. + */ + int UpdateReference(const unsigned char * const * im, bool subsample = true, bool detect_corners = true); + + /*! + * Returns the transformation from the display reference to the alignment reference frame + */ + void Get_H_dref_to_ref(double H[9]); + /*! + * Returns the transformation from the display reference to the inspection reference frame + */ + void Get_H_dref_to_ins(double H[9]); + /*! + * Set the transformation from the display reference to the inspection reference frame + * \param H the transformation to set + */ + void Set_H_dref_to_ins(double H[9]); + + /*! + * Reset the display reference to the current frame. + */ + void ResetDisplayReference(); + + /*! + * Estimate a secondary motion model starting from the specified transformation. + * \param H the primary motion model to start from + */ + void EstimateSecondaryModel(double H[9]); + + /*! + * + */ + void SelectOutliers(); + + char *profile_string; + +protected: + void Clean(); + void GenerateQuarterResImage(const unsigned char* const * im); + + int m_im_width; + int m_im_height; + + // RANSAC and refinement parameters: + int m_homography_type; + int m_max_iterations; + double m_scale; + int m_nr_samples; + int m_chunk_size; + double m_outlier_t2; + + // Whether to fit a linear model to just the inliers at the end + bool m_linear_polish; + double m_polish_C[36]; + double m_polish_D[6]; + + // local state + bool m_current_is_reference; + bool m_initialized; + + // inspection to reference homography: + double m_H_ref_to_ins[9]; + double m_H_dref_to_ref[9]; + + // feature extraction and matching: + db_CornerDetector_u m_cd; + db_Matcher_u m_cm; + + // length of corner arrays: + unsigned long m_max_nr_corners; + + // corner locations of reference image features: + double * m_x_corners_ref; + double * m_y_corners_ref; + int m_nr_corners_ref; + + // corner locations of inspection image features: + double * m_x_corners_ins; + double * m_y_corners_ins; + int m_nr_corners_ins; + + // length of match index arrays: + unsigned long m_max_nr_matches; + + // match indices: + int * m_match_index_ref; + int * m_match_index_ins; + int m_nr_matches; + + // pointer to internal copy of the reference image: + unsigned char ** m_reference_image; + + // pointer to internal copy of last aligned inspection image: + unsigned char ** m_aligned_ins_image; + + // pointer to quarter resolution image, if used. + unsigned char** m_quarter_res_image; + + // temporary storage for the quarter resolution image processing + unsigned char** m_horz_smooth_subsample_image; + + // temporary space for homography computation: + double * m_temp_double; + int * m_temp_int; + + // homogenous image point arrays: + double * m_corners_ref; + double * m_corners_ins; + + // Indices of the points within the match lists + int * m_inlier_indices; + int m_num_inlier_indices; + + //void ComputeInliers(double H[9], std::vector<int> &inlier_indices); + void ComputeInliers(double H[9]); + + // cost arrays: + void ComputeCostArray(); + bool m_sq_cost_computed; + double * m_sq_cost; + + // cost histogram: + void ComputeCostHistogram(); + int *m_cost_histogram; + + void SetOutlierThreshold(); + + // utility function for smoothing the motion parameters. + void SmoothMotion(void); + +private: + double m_K[9]; + const int m_over_allocation; + + bool m_reference_set; + + // Maximum number of inliers seen until now w.r.t the current reference frame + int m_max_inlier_count; + + // Number of cost histogram bins: + int m_nr_bins; + // All costs above this threshold get put into the last bin: + int m_max_cost_pix; + + // whether to quarter the image resolution for processing, or not + bool m_quarter_resolution; + + // the period (in number of frames) for reference update. + unsigned int m_reference_update_period; + + // the number of frames processed so far. + unsigned int m_nr_frames_processed; + + // smoother for motion transformations + db_StabilizationSmoother m_stab_smoother; + + // boolean to control whether motion smoothing occurs (or not) + bool m_do_motion_smoothing; + + // double to set the gain for motion smoothing + double m_motion_smoothing_gain; +}; +/*! + Create look-up tables to undistort images. Only Bougeut (Matlab toolkit) + is currently supported. Can be used with db_WarpImageLut_u(). + \code + xd = H*xs; + xd = xd/xd(3); + \endcode + \param lut_x pre-allocated float image + \param lut_y pre-allocated float image + \param w width + \param h height + \param H image homography from source to destination + */ +inline void db_GenerateHomographyLut(float ** lut_x,float ** lut_y,int w,int h,const double H[9]) +{ + assert(lut_x && lut_y); + double x[3] = {0.0,0.0,1.0}; + double xb[3]; + +/* + double xl[3]; + + // Determine the output coordinate system ROI + double Hinv[9]; + db_InvertAffineTransform(Hinv,H); + db_Multiply3x3_3x1(xl, Hinv, x); + xl[0] = db_SafeDivision(xl[0],xl[2]); + xl[1] = db_SafeDivision(xl[1],xl[2]); +*/ + + for ( int i = 0; i < w; ++i ) + for ( int j = 0; j < h; ++j ) + { + x[0] = double(i); + x[1] = double(j); + db_Multiply3x3_3x1(xb, H, x); + xb[0] = db_SafeDivision(xb[0],xb[2]); + xb[1] = db_SafeDivision(xb[1],xb[2]); + + lut_x[j][i] = float(xb[0]); + lut_y[j][i] = float(xb[1]); + } +} + +/*! + * Perform a look-up table warp for packed RGB ([rgbrgbrgb...]) images. + * The LUTs must be float images of the same size as source image. + * The source value x_s is determined from destination (x_d,y_d) through lut_x + * and y_s is determined from lut_y: + \code + x_s = lut_x[y_d][x_d]; + y_s = lut_y[y_d][x_d]; + \endcode + + * \param src source image (w*3 by h) + * \param dst destination image (w*3 by h) + * \param w width + * \param h height + * \param lut_x LUT for x + * \param lut_y LUT for y + */ +inline void db_WarpImageLutFast_rgb(const unsigned char * const * src, unsigned char ** dst, int w, int h, + const float * const * lut_x, const float * const * lut_y) +{ + assert(src && dst); + int xd=0, yd=0; + + for ( int i = 0; i < w; ++i ) + for ( int j = 0; j < h; ++j ) + { + xd = static_cast<unsigned int>(lut_x[j][i]); + yd = static_cast<unsigned int>(lut_y[j][i]); + if ( xd >= w || yd >= h || + xd < 0 || yd < 0) + { + dst[j][3*i ] = 0; + dst[j][3*i+1] = 0; + dst[j][3*i+2] = 0; + } + else + { + dst[j][3*i ] = src[yd][3*xd ]; + dst[j][3*i+1] = src[yd][3*xd+1]; + dst[j][3*i+2] = src[yd][3*xd+2]; + } + } +} + +inline unsigned char db_BilinearInterpolationRGB(double y, double x, const unsigned char * const * v, int offset) +{ + int floor_x=(int) x; + int floor_y=(int) y; + + int ceil_x=floor_x+1; + int ceil_y=floor_y+1; + + unsigned char f00 = v[floor_y][3*floor_x+offset]; + unsigned char f01 = v[floor_y][3*ceil_x+offset]; + unsigned char f10 = v[ceil_y][3*floor_x+offset]; + unsigned char f11 = v[ceil_y][3*ceil_x+offset]; + + double xl = x-floor_x; + double yl = y-floor_y; + + return (unsigned char)(f00*(1-yl)*(1-xl) + f10*yl*(1-xl) + f01*(1-yl)*xl + f11*yl*xl); +} + +inline void db_WarpImageLutBilinear_rgb(const unsigned char * const * src, unsigned char ** dst, int w, int h, + const float * const * lut_x, const float * const * lut_y) +{ + assert(src && dst); + double xd=0.0, yd=0.0; + + for ( int i = 0; i < w; ++i ) + for ( int j = 0; j < h; ++j ) + { + xd = static_cast<double>(lut_x[j][i]); + yd = static_cast<double>(lut_y[j][i]); + if ( xd > w-2 || yd > h-2 || + xd < 0.0 || yd < 0.0) + { + dst[j][3*i ] = 0; + dst[j][3*i+1] = 0; + dst[j][3*i+2] = 0; + } + else + { + dst[j][3*i ] = db_BilinearInterpolationRGB(yd,xd,src,0); + dst[j][3*i+1] = db_BilinearInterpolationRGB(yd,xd,src,1); + dst[j][3*i+2] = db_BilinearInterpolationRGB(yd,xd,src,2); + } + } +} + +inline double SquaredInhomogenousHomographyError(double y[3],double H[9],double x[3]){ + double x0,x1,x2,mult; + double sd; + + x0=H[0]*x[0]+H[1]*x[1]+H[2]; + x1=H[3]*x[0]+H[4]*x[1]+H[5]; + x2=H[6]*x[0]+H[7]*x[1]+H[8]; + mult=1.0/((x2!=0.0)?x2:1.0); + sd=(y[0]-x0*mult)*(y[0]-x0*mult)+(y[1]-x1*mult)*(y[1]-x1*mult); + + return(sd); +} + + +// functions related to profiling +#if PROFILE + +/* return current time in milliseconds */ +static double +now_ms(void) +{ + //struct timespec res; + struct timeval res; + //clock_gettime(CLOCK_REALTIME, &res); + gettimeofday(&res, NULL); + return 1000.0*res.tv_sec + (double)res.tv_usec/1e3; +} + +#endif |