diff options
Diffstat (limited to 'jni_mosaic/feature_stab/src/dbreg/dbreg.h')
| -rw-r--r-- | jni_mosaic/feature_stab/src/dbreg/dbreg.h | 581 |
1 files changed, 0 insertions, 581 deletions
diff --git a/jni_mosaic/feature_stab/src/dbreg/dbreg.h b/jni_mosaic/feature_stab/src/dbreg/dbreg.h deleted file mode 100644 index 4eb244481..000000000 --- a/jni_mosaic/feature_stab/src/dbreg/dbreg.h +++ /dev/null @@ -1,581 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#pragma once - -#ifdef _WIN32 -#ifdef DBREG_EXPORTS -#define DBREG_API __declspec(dllexport) -#else -#define DBREG_API __declspec(dllimport) -#endif -#else -#define DBREG_API -#endif - -// @jke - the next few lines are for extracting timing data. TODO: Remove after test -#define PROFILE 0 - -#include "dbstabsmooth.h" - -#include <db_feature_detection.h> -#include <db_feature_matching.h> -#include <db_rob_image_homography.h> - -#if PROFILE - #include <sys/time.h> -#endif - -/*! \mainpage db_FrameToReferenceRegistration - - \section intro Introduction - - db_FrameToReferenceRegistration provides a simple interface to a set of sophisticated algorithms for stabilizing - video sequences. As its name suggests, the class is used to compute parameters that will allow us to warp incoming video - frames and register them with respect to a so-called <i>reference</i> frame. The reference frame is simply the first - frame of a sequence; the registration process is that of estimating the parameters of a warp that can be applied to - subsequent frames to make those frames align with the reference. A video made up of these warped frames will be more - stable than the input video. - - For more technical information on the internal structure of the algorithms used within the db_FrameToRegistration class, - please follow this <a href="../Sarnoff image registration.docx">link</a>. - - \section usage Usage - In addition to the class constructor, there are two main functions of db_FrameToReferenceRegistration that are of - interest to the programmer. db_FrameToReferenceRegistration::Init(...) is used to initialize the parameters of the - registration algorithm. db_FrameToReferenceRegistration::AddFrame(...) is the method by which each new video frame - is introduced to the registration algorithm, and produces the estimated registration warp parameters. - - The following example illustrates how the major methods of the class db_FrameToReferenceRegistration can be used together - to calculate the registration parameters for an image sequence. In the example, the calls to the methods of - db_FrameToReferenceRegistration match those found in the API, but supporting code should be considered pseudo-code. - For a more complete example, please consult the source code for dbregtest. - - - \code - // feature-based image registration class: - db_FrameToReferenceRegistration reg; - - // Image data - const unsigned char * const * image_storage; - - // The 3x3 frame to reference registration parameters - double frame_to_ref_homography[9]; - - // a counter to count the number of frames processed. - unsigned long frame_counter; - // ... - - // main loop - keep going while there are images to process. - while (ImagesAreAvailable) - { - // Call functions to place latest data into image_storage - // ... - - // if the registration object is not yet initialized, then do so - // The arguments to this function are explained in the accompanying - // html API documentation - if (!reg.Initialized()) - { - reg.Init(w,h,motion_model_type,25,linear_polish,quarter_resolution, - DB_POINT_STANDARDDEV,reference_update_period, - do_motion_smoothing,motion_smoothing_gain, - DB_DEFAULT_NR_SAMPLES,DB_DEFAULT_CHUNK_SIZE, - nr_corners,max_disparity); - } - - // Present the new image data to the registration algorithm, - // with the result being stored in the frame_to_ref_homography - // variable. - reg.AddFrame(image_storage,frame_to_ref_homography); - - // frame_to_ref_homography now contains the stabilizing transform - // use this to warp the latest image for display, etc. - - // if this is the first frame, we need to tell the registration - // class to store the image as its reference. Otherwise, AddFrame - // takes care of that. - if (frame_counter == 0) - { - reg.UpdateReference(image_storage); - } - - // increment the frame counter - frame_counter++; - } - - \endcode - - */ - -/*! - * Performs feature-based frame to reference image registration. - */ -class DBREG_API db_FrameToReferenceRegistration -{ -public: - db_FrameToReferenceRegistration(void); - ~db_FrameToReferenceRegistration(); - - /*! - * Set parameters and allocate memory. Note: The default values of these parameters have been set to the values used for the android implementation (i.e. the demo APK). - * \param width image width - * \param height image height - * \param homography_type see definitions in \ref LMRobImageHomography - * \param max_iterations max number of polishing steps - * \param linear_polish whether to perform a linear polishing step after RANSAC - * \param quarter_resolution whether to process input images at quarter resolution (for computational efficiency) - * \param scale Cauchy scale coefficient (see db_ExpCauchyReprojectionError() ) - * \param reference_update_period how often to update the alignment reference (in units of number of frames) - * \param do_motion_smoothing whether to perform display reference smoothing - * \param motion_smoothing_gain weight factor to reflect how fast the display reference must follow the current frame if motion smoothing is enabled - * \param nr_samples number of times to compute a hypothesis - * \param chunk_size size of cost chunks - * \param cd_target_nr_corners target number of corners for corner detector - * \param cm_max_disparity maximum disparity search range for corner matcher (in units of ratio of image width) - * \param cm_use_smaller_matching_window if set to true, uses a correlation window of 5x5 instead of the default 11x11 - * \param cd_nr_horz_blocks the number of horizontal blocks for the corner detector to partition the image - * \param cd_nr_vert_blocks the number of vertical blocks for the corner detector to partition the image - */ - void Init(int width, int height, - int homography_type = DB_HOMOGRAPHY_TYPE_DEFAULT, - int max_iterations = DB_DEFAULT_MAX_ITERATIONS, - bool linear_polish = false, - bool quarter_resolution = true, - double scale = DB_POINT_STANDARDDEV, - unsigned int reference_update_period = 3, - bool do_motion_smoothing = false, - double motion_smoothing_gain = 0.75, - int nr_samples = DB_DEFAULT_NR_SAMPLES, - int chunk_size = DB_DEFAULT_CHUNK_SIZE, - int cd_target_nr_corners = 500, - double cm_max_disparity = 0.2, - bool cm_use_smaller_matching_window = false, - int cd_nr_horz_blocks = 5, - int cd_nr_vert_blocks = 5); - - /*! - * Reset the transformation type that is being use to perform alignment. Use this to change the alignment type at run time. - * \param homography_type the type of transformation to use for performing alignment (see definitions in \ref LMRobImageHomography) - */ - void ResetHomographyType(int homography_type) { m_homography_type = homography_type; } - - /*! - * Enable/Disable motion smoothing. Use this to turn motion smoothing on/off at run time. - * \param enable flag indicating whether to turn the motion smoothing on or off. - */ - void ResetSmoothing(bool enable) { m_do_motion_smoothing = enable; } - - /*! - * Align an inspection image to an existing reference image, update the reference image if due and perform motion smoothing if enabled. - * \param im new inspection image - * \param H computed transformation from reference to inspection coordinate frame. Identity is returned if no reference frame was set. - * \param force_reference make this the new reference image - */ - int AddFrame(const unsigned char * const * im, double H[9], bool force_reference=false, bool prewarp=false); - - /*! - * Returns true if Init() was run. - */ - bool Initialized() const { return m_initialized; } - - /*! - * Returns true if the current frame is being used as the alignment reference. - */ - bool IsCurrentReference() const { return m_current_is_reference; } - - /*! - * Returns true if we need to call UpdateReference now. - */ - bool NeedReferenceUpdate(); - - /*! - * Returns the pointer reference to the alignment reference image data - */ - unsigned char ** GetReferenceImage() { return m_reference_image; } - - /*! - * Returns the pointer reference to the double array containing the homogeneous coordinates for the matched reference image corners. - */ - double * GetRefCorners() { return m_corners_ref; } - /*! - * Returns the pointer reference to the double array containing the homogeneous coordinates for the matched inspection image corners. - */ - double * GetInsCorners() { return m_corners_ins; } - /*! - * Returns the number of correspondences between the reference and inspection images. - */ - int GetNrMatches() { return m_nr_matches; } - - /*! - * Returns the number of corners detected in the current reference image. - */ - int GetNrRefCorners() { return m_nr_corners_ref; } - - /*! - * Returns the pointer to an array of indices that were found to be RANSAC inliers from the matched corner lists. - */ - int* GetInliers() { return m_inlier_indices; } - - /*! - * Returns the number of inliers from the RANSAC matching step. - */ - int GetNrInliers() { return m_num_inlier_indices; } - - //std::vector<int>& GetInliers(); - //void Polish(std::vector<int> &inlier_indices); - - /*! - * Perform a linear polishing step by re-estimating the alignment transformation using the RANSAC inliers. - * \param inlier_indices pointer to an array of indices that were found to be RANSAC inliers from the matched corner lists. - * \param num_inlier_indices number of inliers i.e. the length of the array passed as the first argument. - */ - void Polish(int *inlier_indices, int &num_inlier_indices); - - /*! - * Reset the motion smoothing parameters to their initial values. - */ - void ResetMotionSmoothingParameters() { m_stab_smoother.Init(); } - - /*! - * Update the alignment reference image to the specified image. - * \param im pointer to the image data to be used as the new alignment reference. - * \param subsample boolean flag to control whether the function should internally subsample the provided image to the size provided in the Init() function. - */ - int UpdateReference(const unsigned char * const * im, bool subsample = true, bool detect_corners = true); - - /*! - * Returns the transformation from the display reference to the alignment reference frame - */ - void Get_H_dref_to_ref(double H[9]); - /*! - * Returns the transformation from the display reference to the inspection reference frame - */ - void Get_H_dref_to_ins(double H[9]); - /*! - * Set the transformation from the display reference to the inspection reference frame - * \param H the transformation to set - */ - void Set_H_dref_to_ins(double H[9]); - - /*! - * Reset the display reference to the current frame. - */ - void ResetDisplayReference(); - - /*! - * Estimate a secondary motion model starting from the specified transformation. - * \param H the primary motion model to start from - */ - void EstimateSecondaryModel(double H[9]); - - /*! - * - */ - void SelectOutliers(); - - char *profile_string; - -protected: - void Clean(); - void GenerateQuarterResImage(const unsigned char* const * im); - - int m_im_width; - int m_im_height; - - // RANSAC and refinement parameters: - int m_homography_type; - int m_max_iterations; - double m_scale; - int m_nr_samples; - int m_chunk_size; - double m_outlier_t2; - - // Whether to fit a linear model to just the inliers at the end - bool m_linear_polish; - double m_polish_C[36]; - double m_polish_D[6]; - - // local state - bool m_current_is_reference; - bool m_initialized; - - // inspection to reference homography: - double m_H_ref_to_ins[9]; - double m_H_dref_to_ref[9]; - - // feature extraction and matching: - db_CornerDetector_u m_cd; - db_Matcher_u m_cm; - - // length of corner arrays: - unsigned long m_max_nr_corners; - - // corner locations of reference image features: - double * m_x_corners_ref; - double * m_y_corners_ref; - int m_nr_corners_ref; - - // corner locations of inspection image features: - double * m_x_corners_ins; - double * m_y_corners_ins; - int m_nr_corners_ins; - - // length of match index arrays: - unsigned long m_max_nr_matches; - - // match indices: - int * m_match_index_ref; - int * m_match_index_ins; - int m_nr_matches; - - // pointer to internal copy of the reference image: - unsigned char ** m_reference_image; - - // pointer to internal copy of last aligned inspection image: - unsigned char ** m_aligned_ins_image; - - // pointer to quarter resolution image, if used. - unsigned char** m_quarter_res_image; - - // temporary storage for the quarter resolution image processing - unsigned char** m_horz_smooth_subsample_image; - - // temporary space for homography computation: - double * m_temp_double; - int * m_temp_int; - - // homogenous image point arrays: - double * m_corners_ref; - double * m_corners_ins; - - // Indices of the points within the match lists - int * m_inlier_indices; - int m_num_inlier_indices; - - //void ComputeInliers(double H[9], std::vector<int> &inlier_indices); - void ComputeInliers(double H[9]); - - // cost arrays: - void ComputeCostArray(); - bool m_sq_cost_computed; - double * m_sq_cost; - - // cost histogram: - void ComputeCostHistogram(); - int *m_cost_histogram; - - void SetOutlierThreshold(); - - // utility function for smoothing the motion parameters. - void SmoothMotion(void); - -private: - double m_K[9]; - const int m_over_allocation; - - bool m_reference_set; - - // Maximum number of inliers seen until now w.r.t the current reference frame - int m_max_inlier_count; - - // Number of cost histogram bins: - int m_nr_bins; - // All costs above this threshold get put into the last bin: - int m_max_cost_pix; - - // whether to quarter the image resolution for processing, or not - bool m_quarter_resolution; - - // the period (in number of frames) for reference update. - unsigned int m_reference_update_period; - - // the number of frames processed so far. - unsigned int m_nr_frames_processed; - - // smoother for motion transformations - db_StabilizationSmoother m_stab_smoother; - - // boolean to control whether motion smoothing occurs (or not) - bool m_do_motion_smoothing; - - // double to set the gain for motion smoothing - double m_motion_smoothing_gain; -}; -/*! - Create look-up tables to undistort images. Only Bougeut (Matlab toolkit) - is currently supported. Can be used with db_WarpImageLut_u(). - \code - xd = H*xs; - xd = xd/xd(3); - \endcode - \param lut_x pre-allocated float image - \param lut_y pre-allocated float image - \param w width - \param h height - \param H image homography from source to destination - */ -inline void db_GenerateHomographyLut(float ** lut_x,float ** lut_y,int w,int h,const double H[9]) -{ - assert(lut_x && lut_y); - double x[3] = {0.0,0.0,1.0}; - double xb[3]; - -/* - double xl[3]; - - // Determine the output coordinate system ROI - double Hinv[9]; - db_InvertAffineTransform(Hinv,H); - db_Multiply3x3_3x1(xl, Hinv, x); - xl[0] = db_SafeDivision(xl[0],xl[2]); - xl[1] = db_SafeDivision(xl[1],xl[2]); -*/ - - for ( int i = 0; i < w; ++i ) - for ( int j = 0; j < h; ++j ) - { - x[0] = double(i); - x[1] = double(j); - db_Multiply3x3_3x1(xb, H, x); - xb[0] = db_SafeDivision(xb[0],xb[2]); - xb[1] = db_SafeDivision(xb[1],xb[2]); - - lut_x[j][i] = float(xb[0]); - lut_y[j][i] = float(xb[1]); - } -} - -/*! - * Perform a look-up table warp for packed RGB ([rgbrgbrgb...]) images. - * The LUTs must be float images of the same size as source image. - * The source value x_s is determined from destination (x_d,y_d) through lut_x - * and y_s is determined from lut_y: - \code - x_s = lut_x[y_d][x_d]; - y_s = lut_y[y_d][x_d]; - \endcode - - * \param src source image (w*3 by h) - * \param dst destination image (w*3 by h) - * \param w width - * \param h height - * \param lut_x LUT for x - * \param lut_y LUT for y - */ -inline void db_WarpImageLutFast_rgb(const unsigned char * const * src, unsigned char ** dst, int w, int h, - const float * const * lut_x, const float * const * lut_y) -{ - assert(src && dst); - int xd=0, yd=0; - - for ( int i = 0; i < w; ++i ) - for ( int j = 0; j < h; ++j ) - { - xd = static_cast<unsigned int>(lut_x[j][i]); - yd = static_cast<unsigned int>(lut_y[j][i]); - if ( xd >= w || yd >= h || - xd < 0 || yd < 0) - { - dst[j][3*i ] = 0; - dst[j][3*i+1] = 0; - dst[j][3*i+2] = 0; - } - else - { - dst[j][3*i ] = src[yd][3*xd ]; - dst[j][3*i+1] = src[yd][3*xd+1]; - dst[j][3*i+2] = src[yd][3*xd+2]; - } - } -} - -inline unsigned char db_BilinearInterpolationRGB(double y, double x, const unsigned char * const * v, int offset) -{ - int floor_x=(int) x; - int floor_y=(int) y; - - int ceil_x=floor_x+1; - int ceil_y=floor_y+1; - - unsigned char f00 = v[floor_y][3*floor_x+offset]; - unsigned char f01 = v[floor_y][3*ceil_x+offset]; - unsigned char f10 = v[ceil_y][3*floor_x+offset]; - unsigned char f11 = v[ceil_y][3*ceil_x+offset]; - - double xl = x-floor_x; - double yl = y-floor_y; - - return (unsigned char)(f00*(1-yl)*(1-xl) + f10*yl*(1-xl) + f01*(1-yl)*xl + f11*yl*xl); -} - -inline void db_WarpImageLutBilinear_rgb(const unsigned char * const * src, unsigned char ** dst, int w, int h, - const float * const * lut_x, const float * const * lut_y) -{ - assert(src && dst); - double xd=0.0, yd=0.0; - - for ( int i = 0; i < w; ++i ) - for ( int j = 0; j < h; ++j ) - { - xd = static_cast<double>(lut_x[j][i]); - yd = static_cast<double>(lut_y[j][i]); - if ( xd > w-2 || yd > h-2 || - xd < 0.0 || yd < 0.0) - { - dst[j][3*i ] = 0; - dst[j][3*i+1] = 0; - dst[j][3*i+2] = 0; - } - else - { - dst[j][3*i ] = db_BilinearInterpolationRGB(yd,xd,src,0); - dst[j][3*i+1] = db_BilinearInterpolationRGB(yd,xd,src,1); - dst[j][3*i+2] = db_BilinearInterpolationRGB(yd,xd,src,2); - } - } -} - -inline double SquaredInhomogenousHomographyError(double y[3],double H[9],double x[3]){ - double x0,x1,x2,mult; - double sd; - - x0=H[0]*x[0]+H[1]*x[1]+H[2]; - x1=H[3]*x[0]+H[4]*x[1]+H[5]; - x2=H[6]*x[0]+H[7]*x[1]+H[8]; - mult=1.0/((x2!=0.0)?x2:1.0); - sd=(y[0]-x0*mult)*(y[0]-x0*mult)+(y[1]-x1*mult)*(y[1]-x1*mult); - - return(sd); -} - - -// functions related to profiling -#if PROFILE - -/* return current time in milliseconds */ -static double -now_ms(void) -{ - //struct timespec res; - struct timeval res; - //clock_gettime(CLOCK_REALTIME, &res); - gettimeofday(&res, NULL); - return 1000.0*res.tv_sec + (double)res.tv_usec/1e3; -} - -#endif |