Fixed QP lockups

Fixed QP getting locked up at particluar values even when
it is supposed to be changing by ensuring that Qp will have
a swing of atleast +-2 from previous value and correcting
the Qp translation tables

Change-Id: Id6f0fa4f282c288b81cfe278f34957a0fa68ff8c

2 files changed, 103 insertions, 99 deletions

diff --git a/encoder/ih264e_globals.c b/encoder/ih264e_globals.c
index 28d3faf..6719c5f 100644
--- a/encoder/ih264e_globals.c
+++ b/encoder/ih264e_globals.c
@@ -225,11 +225,11 @@ const WORD8 gi1_mv_pred_condition[8] =
  * QPm = a + b*2^(QPh/6)
  *
  * Appling boundary condition that
- *      1) QPm = 1 if QPh = 0
- *      2) QPm = 228 if QPh = 51,
+ *      1) QPm = 0.625 if QPh = 0
+ *      2) QPm =   224 if QPh = 51,
  *
  * we will have
- *  a = -0.372, b = 0.628
+ *  a = 0.0063, b = 0.6187
  *
  * Hence the relatiohship is
  *  QPm = a + b*2^(Qph/6)
@@ -238,101 +238,96 @@ const WORD8 gi1_mv_pred_condition[8] =
  *
  * Unrounded values for gau1_h264_to_mpeg2_qmap[H264_QP_ELEM] =
  *
- *   0.33291     0.41923     0.51613     0.62489     0.74697     0.88400
- *   1.03781     1.21046     1.40425     1.62178     1.86594     2.14000
- *   2.44762     2.79292     3.18050     3.61555     4.10388     4.65200
- *   5.26725     5.95784     6.73301     7.60310     8.57975     9.67600
- *   10.90650    12.28769    13.83802    15.57821    17.53150    19.72400
- *   22.18500    24.94737    28.04804    31.52841    35.43500    39.82000
- *   44.74199    50.26675    56.46807    63.42882    71.24200    80.01200
- *   89.85599    100.90549   113.30814   127.22965   142.85601   160.39600
- *   180.08398   202.18299   226.98829
+ *   0.625       0.70077     0.78581     0.88127     0.98843     1.10870
+ *   1.24370     1.39523     1.56533     1.75625     1.97055     2.21110
+ *   2.48110     2.78417     3.12435     3.50620     3.93480     4.41589
+ *   4.95590     5.56204     6.24241     7.00609     7.86330     8.82548
+ *   9.90550     11.11778    12.47851    14.00588    15.72030    17.64467
+ *   19.80470    22.22925    24.95072    28.00547    31.43430    35.28304
+ *   39.60310    44.45221    49.89514    56.00463    62.86230    70.55978
+ *   79.19990    88.89811    99.78398    112.00296   125.71830   141.11325
+ *   158.39350   177.78992   199.56167   223.99963
+ *
  *
- * Unrounded values for gau1_mpeg2_to_h264_qmap[MPEG2_QP_ELEM]
  *
- *  -4.5328    6.7647   11.5036   14.5486   16.7967   18.5797   20.0575
- *  21.3193   22.4204   23.3971   24.2747   25.0715   25.8010   26.4738
- *  27.0981   27.6804   28.2259   28.7391   29.2236   29.6824   30.1181
- *  30.5329   30.9287   31.3072   31.6699   32.0180   32.3526   32.6748
- *  32.9854   33.2852   33.5750   33.8554   34.1270   34.3904   34.6460
- *  34.8942   35.1355   35.3703   35.5989   35.8216   36.0387   36.2505
- *  36.4572   36.6591   36.8564   37.0494   37.2381   37.4228   37.6036
- *  37.7807   37.9543   38.1244   38.2913   38.4550   38.6157   38.7735
- *  38.9284   39.0806   39.2302   39.3772   39.5218   39.6640   39.8039
- *  39.9416   40.0771   40.2106   40.3420   40.4714   40.5990   40.7247
- *  40.8486   40.9707   41.0911   41.2099   41.3271   41.4427   41.5568
- *  41.6694   41.7806   41.8903   41.9987   42.1057   42.2115   42.3159
- *  42.4191   42.5211   42.6219   42.7216   42.8201   42.9175   43.0138
- *  43.1091   43.2033   43.2965   43.3887   43.4799   43.5702   43.6596
- *  43.7480   43.8356   43.9223   44.0081   44.0930   44.1772   44.2605
- *  44.3431   44.4248   44.5058   44.5861   44.6656   44.7444   44.8224
- *  44.8998   44.9765   45.0525   45.1279   45.2026   45.2766   45.3501
- *  45.4229   45.4951   45.5667   45.6378   45.7082   45.7781   45.8474
- *  45.9162   45.9844   46.0521   46.1193   46.1859   46.2521   46.3177
- *  46.3829   46.4475   46.5117   46.5754   46.6386   46.7014   46.7638
- *  46.8256   46.8871   46.9481   47.0087   47.0689   47.1286   47.1880
- *  47.2469   47.3054   47.3636   47.4213   47.4787   47.5357   47.5923
- *  47.6486   47.7045   47.7600   47.8152   47.8700   47.9245   47.9787
- *  48.0325   48.0859   48.1391   48.1919   48.2444   48.2966   48.3485
- *  48.4000   48.4513   48.5022   48.5529   48.6033   48.6533   48.7031
- *  48.7526   48.8018   48.8508   48.8995   48.9478   48.9960   49.0438
- *  49.0914   49.1388   49.1858   49.2327   49.2792   49.3256   49.3716
- *  49.4175   49.4630   49.5084   49.5535   49.5984   49.6430   49.6875
- *  49.7317   49.7756   49.8194   49.8629   49.9062   49.9493   49.9922
- *  50.0348   50.0773   50.1196   50.1616   50.2034   50.2451   50.2865
- *  50.3278   50.3688   50.4097   50.4503   50.4908   50.5311   50.5712
- *  50.6111   50.6508   50.6904   50.7298   50.7690   50.8080   50.8468
- *  50.8855   50.9240   50.9623   51.0004   51.0384
+ * Unrounded values for gau1_mpeg2_to_h264_qmap[MPEG2_QP_ELEM]
  *
+ *   0         4.1014    10.1288   13.6477   16.1425   18.0768   19.6568
+ *   20.9925   22.1493   23.1696   24.0822   24.9078   25.6614   26.3546
+ *   26.9964   27.5938   28.1527   28.6777   29.1726   29.6408   30.0850
+ *   30.5074   30.9102   31.2951   31.6636   32.0171   32.3567   32.6834
+ *   32.9983   33.3021   33.5957   33.8795   34.1544   34.4208   34.6793
+ *   34.9303   35.1742   35.4114   35.6423   35.8671   36.0863   36.3001
+ *   36.5087   36.7124   36.9115   37.1060   37.2963   37.4825   37.6648
+ *   37.8433   38.0182   38.1896   38.3577   38.5226   38.6844   38.8433
+ *   38.9993   39.1525   39.3031   39.4511   39.5966   39.7397   39.8804
+ *   40.0189   40.1553   40.2895   40.4217   40.5518   40.6801   40.8065
+ *   40.9310   41.0538   41.1749   41.2943   41.4121   41.5283   41.6430
+ *   41.7561   41.8678   41.9781   42.0870   42.1946   42.3008   42.4057
+ *   42.5094   42.6118   42.7131   42.8132   42.9121   43.0099   43.1066
+ *   43.2023   43.2969   43.3905   43.4831   43.5747   43.6653   43.7550
+ *   43.8438   43.9317   44.0187   44.1049   44.1901   44.2746   44.3582
+ *   44.4411   44.5231   44.6044   44.6849   44.7647   44.8438   44.9221
+ *   44.9998   45.0767   45.1530   45.2286   45.3035   45.3779   45.4515
+ *   45.5246   45.5970   45.6689   45.7401   45.8108   45.8809   45.9504
+ *   46.0194   46.0878   46.1557   46.2231   46.2899   46.3563   46.4221
+ *   46.4874   46.5523   46.6166   46.6805   46.7439   46.8069   46.8694
+ *   46.9314   46.9930   47.0542   47.1150   47.1753   47.2352   47.2947
+ *   47.3538   47.4125   47.4708   47.5287   47.5862   47.6433   47.7001
+ *   47.7565   47.8125   47.8682   47.9235   47.9785   48.0331   48.0874
+ *   48.1413   48.1949   48.2482   48.3011   48.3537   48.4060   48.4580
+ *   48.5097   48.5611   48.6122   48.6629   48.7134   48.7636   48.8135
+ *   48.8631   48.9124   48.9615   49.0102   49.0587   49.1069   49.1549
+ *   49.2026   49.2500   49.2972   49.3441   49.3908   49.4372   49.4834
+ *   49.5293   49.5750   49.6204   49.6656   49.7106   49.7553   49.7998
+ *   49.8441   49.8882   49.9320   49.9756   50.0190   50.0622   50.1051
+ *   50.1479   50.1904   50.2327   50.2749   50.3168   50.3585   50.4000
+ *   50.4413   50.4825   50.5234   50.5641   50.6047   50.6450   50.6852
+ *   50.7252   50.7650   50.8046   50.8440   50.8833   50.9224   50.9613
+ *   51.0000
  */
 
 const UWORD8 gau1_h264_to_mpeg2_qmap[H264_QP_ELEM] =
 {
-                 1,     1,     1,     1,     1,     1,
-                 1,     1,     1,     2,     2,     2,
-                 2,     3,     3,     4,     4,     5,
-                 5,     6,     7,     8,     9,     10,
-                 11,    12,    14,    16,    18,    20,
-                 22,    25,    28,    32,    35,    40,
-                 45,    50,    56,    63,    71,    80,
-                 90,    101,   113,   127,   143,   160,
-                 180,   202,   227
+     1,    1,    1,    1,   1,    1,    1,   1,
+     2,    2,    2,    2,   2,    3,    3,   4,
+     4,    4,    5,    6,   6,    7,    8,   9,
+     10,   11,   12,   14,  16,   18,   20,  22,
+     25,   28,   31,   35,  40,   44,   50,  56,
+     63,   71,   79,   89,  100,  112,  126, 141,
+     158,  178,  200,  224
 };
 
 const UWORD8 gau1_mpeg2_to_h264_qmap[MPEG2_QP_ELEM] =
 {
-                 0,    7,    12,   15,   17,   19,   20,
-                 21,   22,   23,   24,   25,   26,   26,
-                 27,   28,   28,   29,   29,   30,   30,
-                 31,   31,   31,   32,   32,   32,   33,
-                 33,   33,   34,   34,   34,   34,   35,
-                 35,   35,   35,   36,   36,   36,   36,
-                 36,   37,   37,   37,   37,   37,   38,
-                 38,   38,   38,   38,   38,   39,   39,
-                 39,   39,   39,   39,   40,   40,   40,
-                 40,   40,   40,   40,   40,   41,   41,
-                 41,   41,   41,   41,   41,   41,   42,
-                 42,   42,   42,   42,   42,   42,   42,
-                 42,   43,   43,   43,   43,   43,   43,
-                 43,   43,   43,   43,   43,   44,   44,
-                 44,   44,   44,   44,   44,   44,   44,
-                 44,   44,   45,   45,   45,   45,   45,
-                 45,   45,   45,   45,   45,   45,   45,
-                 45,   45,   46,   46,   46,   46,   46,
-                 46,   46,   46,   46,   46,   46,   46,
-                 46,   46,   47,   47,   47,   47,   47,
-                 47,   47,   47,   47,   47,   47,   47,
-                 47,   47,   47,   47,   47,   48,   48,
-                 48,   48,   48,   48,   48,   48,   48,
-                 48,   48,   48,   48,   48,   48,   48,
-                 48,   48,   49,   49,   49,   49,   49,
-                 49,   49,   49,   49,   49,   49,   49,
-                 49,   49,   49,   49,   49,   49,   49,
-                 49,   49,   50,   50,   50,   50,   50,
-                 50,   50,   50,   50,   50,   50,   50,
-                 50,   50,   50,   50,   50,   50,   50,
-                 50,   50,   50,   50,   50,   51,   51,
-                 51,   51,   51,   51,   51,   51,   51,
-                 51,   51,   51,   51,   51
+     0,    4,    10,  14,   16,   18,  20,  21,
+     22,   23,   24,  25,   26,   26,  27,  28,
+     28,   29,   29,  30,   30,   31,  31,  31,
+     32,   32,   32,  33,   33,   33,  34,  34,
+     34,   34,   35,  35,   35,   35,  36,  36,
+     36,   36,   37,  37,   37,   37,  37,  37,
+     38,   38,   38,  38,   38,   39,  39,  39,
+     39,   39,   39,  39,   40,   40,  40,  40,
+     40,   40,   40,  41,   41,   41,  41,  41,
+     41,   41,   41,  42,   42,   42,  42,  42,
+     42,   42,   42,  42,   43,   43,  43,  43,
+     43,   43,   43,  43,   43,   43,  43,  44,
+     44,   44,   44,  44,   44,   44,  44,  44,
+     44,   44,   45,  45,   45,   45,  45,  45,
+     45,   45,   45,  45,   45,   45,  45,  46,
+     46,   46,   46,  46,   46,   46,  46,  46,
+     46,   46,   46,  46,   46,   46,  47,  47,
+     47,   47,   47,  47,   47,   47,  47,  47,
+     47,   47,   47,  47,   47,   47,  48,  48,
+     48,   48,   48,  48,   48,   48,  48,  48,
+     48,   48,   48,  48,   48,   48,  48,  48,
+     49,   49,   49,  49,   49,   49,  49,  49,
+     49,   49,   49,  49,   49,   49,  49,  49,
+     49,   49,   49,  49,   49,   50,  50,  50,
+     50,   50,   50,  50,   50,   50,  50,  50,
+     50,   50,   50,  50,   50,   50,  50,  50,
+     50,   50,   50,  50,   51,   51,  51,  51,
+     51,   51,   51,  51,   51,   51,  51,  51,
+     51
 };
 
diff --git a/encoder/irc_rate_control_api.c b/encoder/irc_rate_control_api.c
index f72597a..9693a7c 100644
--- a/encoder/irc_rate_control_api.c
+++ b/encoder/irc_rate_control_api.c
@@ -43,6 +43,10 @@
 #include "irc_rate_control_api_structs.h"
 #include "irc_trace_support.h"
 
+
+#define MIN(a,b)   (((a) < (b)) ? (a) : (b))
+#define MAX(a,b)   (((a) > (b)) ? (a) : (b))
+
 #define DEV_Q   4       /*Q format(Shift) for Deviation range factor */
 #define HI_DEV_FCTR     22  /* 1.4*16 */
 #define LO_DEV_FCTR     12  /* 0.75*16 */
@@ -599,20 +603,25 @@ UWORD8 irc_get_frame_level_qp(rate_control_api_t *ps_rate_control_api,
                     }
                 }
 
-                hi_dev_qp = GET_HI_DEV_QP(prev_qp);
                 /*
-                 * For lower QPs due to scale factor and fixed point arithmetic,
-                 * the hi_dev_qp can be same as that of the prev qp and in which
-                 * case it gets stuck in the lower most qp and thus not allowing
-                 * QPs not to change. To avoid this,for lower qps the hi_dev_qp
-                 * should be made slightly more than prev_qp
+                 * Due to the inexact nature of translation tables, QP may
+                 * get locked at some values. This is because of the inexactness of
+                 * the tables causing a change of +-1 in back and forth translations.
+                 * In that case, if we restrict the QP swing to +-1, we will get
+                 * the lock up condition. Hence we make it such that we will have
+                 * a swing of atleast +- 2 from prev_qp
                  */
-                if(prev_qp == hi_dev_qp)
-                {
-                    hi_dev_qp += 1;
-                }
+
                 lo_dev_qp = GET_LO_DEV_QP(prev_qp);
-                u1_frame_qp = (UWORD8)CLIP_QP((WORD32)u1_frame_qp, hi_dev_qp, lo_dev_qp);
+                lo_dev_qp = MIN(lo_dev_qp, prev_qp - 2);
+                lo_dev_qp = MAX(lo_dev_qp, ps_rate_control_api->au1_min_max_qp[(e_pic_type << 1)]);
+
+                hi_dev_qp = GET_HI_DEV_QP(prev_qp);
+                hi_dev_qp = MAX(hi_dev_qp, prev_qp + 2);
+                hi_dev_qp = MIN(hi_dev_qp, ps_rate_control_api->au1_min_max_qp[(e_pic_type << 1) + 1]);
+
+                u1_frame_qp = (UWORD8)CLIP_QP((WORD32)u1_frame_qp, hi_dev_qp , lo_dev_qp);
+
             }
             else
             {