Set error skip ctbs as multiple 8x8 pus

Bug: 65123471

This is required for incomplete ctbs at the frame
boundaries

Change-Id: I7e41a3ac2f6e35a929ba4ff3ca4cfcc859a7b867
CVE-2017-13185

1 files changed, 94 insertions, 88 deletions

diff --git a/decoder/ihevcd_parse_slice.c b/decoder/ihevcd_parse_slice.c
index 3ff5afa..e282e30 100644
--- a/decoder/ihevcd_parse_slice.c
+++ b/decoder/ihevcd_parse_slice.c
@@ -2177,6 +2177,98 @@ IHEVCD_ERROR_T  ihevcd_parse_sao(codec_t *ps_codec)
  *******************************************************************************
  *
  * @brief
+ *  Set ctb skip
+ *
+ * @par Description:
+ *  During error, sets tu and pu params of a ctb as skip.
+ *
+ * @param[in] ps_codec
+ *  Pointer to codec context
+ *
+ * @returns  None
+ *
+ * @remarks
+ *
+ *
+ *******************************************************************************
+ */
+void ihevcd_set_ctb_skip(codec_t *ps_codec)
+{
+    tu_t *ps_tu;
+    pu_t *ps_pu;
+    sps_t *ps_sps = ps_codec->s_parse.ps_sps;
+    WORD32 ctb_size = 1 << ps_sps->i1_log2_ctb_size;
+    WORD32 ctb_skip_wd, ctb_skip_ht;
+    WORD32 rows_remaining, cols_remaining;
+    WORD32 tu_abs_x, tu_abs_y;
+    WORD32 numbytes_row =  (ps_sps->i2_pic_width_in_luma_samples + 63) / 64;
+    UWORD8 *pu1_pic_intra_flag;
+    UWORD32 u4_mask;
+    WORD32 pu_x,pu_y;
+
+    /* Set pu wd and ht based on whether the ctb is complete or not */
+    rows_remaining = ps_sps->i2_pic_height_in_luma_samples
+                    - (ps_codec->s_parse.i4_ctb_y << ps_sps->i1_log2_ctb_size);
+    ctb_skip_ht = MIN(ctb_size, rows_remaining);
+
+    cols_remaining = ps_sps->i2_pic_width_in_luma_samples
+                    - (ps_codec->s_parse.i4_ctb_x << ps_sps->i1_log2_ctb_size);
+    ctb_skip_wd = MIN(ctb_size, cols_remaining);
+
+    ps_codec->s_parse.s_cu.i4_pred_mode = PRED_MODE_SKIP;
+    ps_codec->s_parse.s_cu.i4_part_mode = PART_2Nx2N;
+
+    for (pu_y = 0; pu_y < ctb_skip_ht ; pu_y += MIN_CU_SIZE)
+    {
+        for (pu_x = 0; pu_x < ctb_skip_wd ; pu_x += MIN_CU_SIZE)
+        {
+            ps_tu = ps_codec->s_parse.ps_tu;
+            ps_tu->b1_cb_cbf = 0;
+            ps_tu->b1_cr_cbf = 0;
+            ps_tu->b1_y_cbf = 0;
+            ps_tu->b4_pos_x = pu_x >> 2;
+            ps_tu->b4_pos_y = pu_y >> 2;
+            ps_tu->b1_transquant_bypass = 0;
+            ps_tu->b3_size = 1;
+            ps_tu->b7_qp = ps_codec->s_parse.u4_qp;
+            ps_tu->b3_chroma_intra_mode_idx = INTRA_PRED_CHROMA_IDX_NONE;
+            ps_tu->b6_luma_intra_mode   = INTRA_PRED_NONE;
+            ps_tu->b1_first_tu_in_cu = 1;
+
+            ps_codec->s_parse.ps_tu++;
+            ps_codec->s_parse.s_cu.i4_tu_cnt++;
+            ps_codec->s_parse.i4_pic_tu_idx++;
+
+            tu_abs_x = (ps_codec->s_parse.i4_ctb_x << ps_sps->i1_log2_ctb_size) + pu_x;
+            tu_abs_y = (ps_codec->s_parse.i4_ctb_y << ps_sps->i1_log2_ctb_size) + pu_y;
+            pu1_pic_intra_flag = ps_codec->s_parse.pu1_pic_intra_flag;
+            pu1_pic_intra_flag += (tu_abs_y >> 3) * numbytes_row;
+            pu1_pic_intra_flag += (tu_abs_x >> 6);
+            u4_mask = (LSB_ONES((MIN_CU_SIZE >> 3)) << (((tu_abs_x) / 8) % 8));
+            u4_mask = ~u4_mask;
+            *pu1_pic_intra_flag &= u4_mask;
+
+            ps_pu = ps_codec->s_parse.ps_pu;
+            ps_pu->b2_part_idx = 0;
+            ps_pu->b4_pos_x = pu_x >> 2;
+            ps_pu->b4_pos_y = pu_y >> 2;
+            ps_pu->b4_wd = 1;
+            ps_pu->b4_ht = 1;
+            ps_pu->b1_intra_flag = 0;
+            ps_pu->b3_part_mode = ps_codec->s_parse.s_cu.i4_part_mode;
+            ps_pu->b1_merge_flag = 1;
+            ps_pu->b3_merge_idx = 0;
+
+            ps_codec->s_parse.ps_pu++;
+            ps_codec->s_parse.i4_pic_pu_idx++;
+        }
+    }
+}
+
+/**
+ *******************************************************************************
+ *
+ * @brief
  *  Parses Slice data syntax
  *
  * @par Description:
@@ -2644,19 +2736,8 @@ IHEVCD_ERROR_T ihevcd_parse_slice_data(codec_t *ps_codec)
             if (ret != (IHEVCD_ERROR_T)IHEVCD_SUCCESS)
             {
                 /* Reset tu and pu parameters, and signal current ctb as skip */
-                WORD32 pu_skip_wd, pu_skip_ht;
-                WORD32 rows_remaining, cols_remaining;
                 WORD32 tu_coeff_data_reset_size;
 
-                /* Set pu wd and ht based on whether the ctb is complete or not */
-                rows_remaining = ps_sps->i2_pic_height_in_luma_samples
-                                - (ps_codec->s_parse.i4_ctb_y << ps_sps->i1_log2_ctb_size);
-                pu_skip_ht = MIN(ctb_size, rows_remaining);
-
-                cols_remaining = ps_sps->i2_pic_width_in_luma_samples
-                                - (ps_codec->s_parse.i4_ctb_x << ps_sps->i1_log2_ctb_size);
-                pu_skip_wd = MIN(ctb_size, cols_remaining);
-
                 ps_codec->s_parse.ps_tu = ps_tu;
                 ps_codec->s_parse.s_cu.i4_tu_cnt = i4_tu_cnt;
                 ps_codec->s_parse.i4_pic_tu_idx = i4_pic_tu_idx;
@@ -2664,41 +2745,11 @@ IHEVCD_ERROR_T ihevcd_parse_slice_data(codec_t *ps_codec)
                 ps_codec->s_parse.ps_pu = ps_pu;
                 ps_codec->s_parse.i4_pic_pu_idx = i4_pic_pu_idx;
 
-                ps_tu->b1_cb_cbf = 0;
-                ps_tu->b1_cr_cbf = 0;
-                ps_tu->b1_y_cbf = 0;
-                ps_tu->b4_pos_x = 0;
-                ps_tu->b4_pos_y = 0;
-                ps_tu->b1_transquant_bypass = 0;
-                ps_tu->b3_size = (ps_sps->i1_log2_ctb_size - 2);
-                ps_tu->b7_qp = ps_codec->s_parse.u4_qp;
-                ps_tu->b3_chroma_intra_mode_idx = INTRA_PRED_CHROMA_IDX_NONE;
-                ps_tu->b6_luma_intra_mode   = INTRA_PRED_NONE;
-                ps_tu->b1_first_tu_in_cu = 1;
-
                 tu_coeff_data_reset_size = (UWORD8 *)ps_codec->s_parse.pv_tu_coeff_data - pu1_tu_coeff_data;
                 memset(pu1_tu_coeff_data, 0, tu_coeff_data_reset_size);
                 ps_codec->s_parse.pv_tu_coeff_data = (void *)pu1_tu_coeff_data;
 
-                ps_codec->s_parse.ps_tu++;
-                ps_codec->s_parse.s_cu.i4_tu_cnt++;
-                ps_codec->s_parse.i4_pic_tu_idx++;
-
-                ps_codec->s_parse.s_cu.i4_pred_mode = PRED_MODE_SKIP;
-                ps_codec->s_parse.s_cu.i4_part_mode = PART_2Nx2N;
-
-                ps_pu->b2_part_idx = 0;
-                ps_pu->b4_pos_x = 0;
-                ps_pu->b4_pos_y = 0;
-                ps_pu->b4_wd = (pu_skip_wd >> 2) - 1;
-                ps_pu->b4_ht = (pu_skip_ht >> 2) - 1;
-                ps_pu->b1_intra_flag = 0;
-                ps_pu->b3_part_mode = ps_codec->s_parse.s_cu.i4_part_mode;
-                ps_pu->b1_merge_flag = 1;
-                ps_pu->b3_merge_idx = 0;
-
-                ps_codec->s_parse.ps_pu++;
-                ps_codec->s_parse.i4_pic_pu_idx++;
+                ihevcd_set_ctb_skip(ps_codec);
 
                 /* Set slice error to suppress further parsing and
                  * signal end of slice.
@@ -2710,52 +2761,7 @@ IHEVCD_ERROR_T ihevcd_parse_slice_data(codec_t *ps_codec)
         }
         else
         {
-            tu_t *ps_tu = ps_codec->s_parse.ps_tu;
-            pu_t *ps_pu = ps_codec->s_parse.ps_pu;
-            WORD32 pu_skip_wd, pu_skip_ht;
-            WORD32 rows_remaining, cols_remaining;
-
-            /* Set pu wd and ht based on whether the ctb is complete or not */
-            rows_remaining = ps_sps->i2_pic_height_in_luma_samples
-                            - (ps_codec->s_parse.i4_ctb_y << ps_sps->i1_log2_ctb_size);
-            pu_skip_ht = MIN(ctb_size, rows_remaining);
-
-            cols_remaining = ps_sps->i2_pic_width_in_luma_samples
-                            - (ps_codec->s_parse.i4_ctb_x << ps_sps->i1_log2_ctb_size);
-            pu_skip_wd = MIN(ctb_size, cols_remaining);
-
-            ps_tu->b1_cb_cbf = 0;
-            ps_tu->b1_cr_cbf = 0;
-            ps_tu->b1_y_cbf = 0;
-            ps_tu->b4_pos_x = 0;
-            ps_tu->b4_pos_y = 0;
-            ps_tu->b1_transquant_bypass = 0;
-            ps_tu->b3_size = (ps_sps->i1_log2_ctb_size - 2);
-            ps_tu->b7_qp = ps_codec->s_parse.u4_qp;
-            ps_tu->b3_chroma_intra_mode_idx = INTRA_PRED_CHROMA_IDX_NONE;
-            ps_tu->b6_luma_intra_mode   = INTRA_PRED_NONE;
-            ps_tu->b1_first_tu_in_cu = 1;
-
-            ps_codec->s_parse.ps_tu++;
-            ps_codec->s_parse.s_cu.i4_tu_cnt++;
-            ps_codec->s_parse.i4_pic_tu_idx++;
-
-            ps_codec->s_parse.s_cu.i4_pred_mode = PRED_MODE_SKIP;
-            ps_codec->s_parse.s_cu.i4_part_mode = PART_2Nx2N;
-
-            ps_pu->b2_part_idx = 0;
-            ps_pu->b4_pos_x = 0;
-            ps_pu->b4_pos_y = 0;
-            ps_pu->b4_wd = (pu_skip_wd >> 2) - 1;
-            ps_pu->b4_ht = (pu_skip_ht >> 2) - 1;
-            ps_pu->b1_intra_flag = 0;
-            ps_pu->b3_part_mode = ps_codec->s_parse.s_cu.i4_part_mode;
-            ps_pu->b1_merge_flag = 1;
-            ps_pu->b3_merge_idx = 0;
-
-            ps_codec->s_parse.ps_pu++;
-            ps_codec->s_parse.i4_pic_pu_idx++;
-
+            ihevcd_set_ctb_skip(ps_codec);
         }
 
         if(0 == ps_codec->i4_slice_error)