Added error check for output buffer size.

The output buffer size given by the application, needs to be checked
in every process call. This is required in the case of resolution
change.

Bug: 36006815
Test: avcdec -i poc.bin
Change-Id: I16a92cdad23eb7b1e12c1a67c1b2599204f29249

2 files changed, 106 insertions, 44 deletions

diff --git a/decoder/ih264d_api.c b/decoder/ih264d_api.c
index 6e81623..f244f4c 100644
--- a/decoder/ih264d_api.c
+++ b/decoder/ih264d_api.c
@@ -2571,6 +2571,89 @@ UWORD32 ih264d_map_error(UWORD32 i4_err_status)
 
 }
 
+UWORD32 ih264d_get_outbuf_size(WORD32 pic_wd,
+                               UWORD32 pic_ht,
+                               UWORD8 u1_chroma_format,
+                               UWORD32 *p_buf_size)
+{
+    UWORD32 u4_min_num_out_bufs = 0;
+
+    if(u1_chroma_format == IV_YUV_420P)
+        u4_min_num_out_bufs = MIN_OUT_BUFS_420;
+    else if(u1_chroma_format == IV_YUV_422ILE)
+        u4_min_num_out_bufs = MIN_OUT_BUFS_422ILE;
+    else if(u1_chroma_format == IV_RGB_565)
+        u4_min_num_out_bufs = MIN_OUT_BUFS_RGB565;
+    else if((u1_chroma_format == IV_YUV_420SP_UV)
+                    || (u1_chroma_format == IV_YUV_420SP_VU))
+        u4_min_num_out_bufs = MIN_OUT_BUFS_420SP;
+
+    if(u1_chroma_format == IV_YUV_420P)
+    {
+        p_buf_size[0] = (pic_wd * pic_ht);
+        p_buf_size[1] = (pic_wd * pic_ht) >> 2;
+        p_buf_size[2] = (pic_wd * pic_ht) >> 2;
+    }
+    else if(u1_chroma_format == IV_YUV_422ILE)
+    {
+        p_buf_size[0] = (pic_wd * pic_ht) * 2;
+        p_buf_size[1] = p_buf_size[2] = 0;
+    }
+    else if(u1_chroma_format == IV_RGB_565)
+    {
+        p_buf_size[0] = (pic_wd * pic_ht) * 2;
+        p_buf_size[1] = p_buf_size[2] = 0;
+    }
+    else if((u1_chroma_format == IV_YUV_420SP_UV)
+                    || (u1_chroma_format == IV_YUV_420SP_VU))
+    {
+        p_buf_size[0] = (pic_wd * pic_ht);
+        p_buf_size[1] = (pic_wd * pic_ht) >> 1;
+        p_buf_size[2] = 0;
+    }
+
+    return u4_min_num_out_bufs;
+}
+
+WORD32 check_app_out_buf_size(dec_struct_t *ps_dec)
+{
+    UWORD32 au4_min_out_buf_size[IVD_VIDDEC_MAX_IO_BUFFERS];
+    UWORD32 u4_min_num_out_bufs, i;
+    UWORD32 pic_wd, pic_ht;
+
+    if(0 == ps_dec->u4_share_disp_buf)
+    {
+        pic_wd = ps_dec->u2_disp_width;
+        pic_ht = ps_dec->u2_disp_height;
+
+    }
+    else
+    {
+        /* In case of shared mode, do not check validity of ps_dec->ps_out_buffer */
+        return (IV_SUCCESS);
+    }
+
+    if(ps_dec->u4_app_disp_width > pic_wd)
+        pic_wd = ps_dec->u4_app_disp_width;
+
+    u4_min_num_out_bufs = ih264d_get_outbuf_size(pic_wd, pic_ht,
+                                                 ps_dec->u1_chroma_format,
+                                                 &au4_min_out_buf_size[0]);
+
+    if(ps_dec->ps_out_buffer->u4_num_bufs < u4_min_num_out_bufs)
+        return IV_FAIL;
+
+    for(i = 0; i < u4_min_num_out_bufs; i++)
+    {
+        if(ps_dec->ps_out_buffer->u4_min_out_buf_size[i]
+                        < au4_min_out_buf_size[i])
+            return (IV_FAIL);
+    }
+
+    return (IV_SUCCESS);
+}
+
+
 /*****************************************************************************/
 /*                                                                           */
 /*  Function Name :  ih264d_video_decode                                     */
@@ -2815,6 +2898,13 @@ WORD32 ih264d_video_decode(iv_obj_t *dec_hdl, void *pv_api_ip, void *pv_api_op)
                                       &(ps_dec->s_disp_op));
         if(0 == ps_dec->s_disp_op.u4_error_code)
         {
+            /* check output buffer size given by the application */
+            if(check_app_out_buf_size(ps_dec) != IV_SUCCESS)
+            {
+                ps_dec_op->u4_error_code= IVD_DISP_FRM_ZERO_OP_BUF_SIZE;
+                return (IV_FAIL);
+            }
+
             ps_dec->u4_fmt_conv_cur_row = 0;
             ps_dec->u4_fmt_conv_num_rows = ps_dec->s_disp_frame_info.u4_y_ht;
             ih264d_format_convert(ps_dec, &(ps_dec->s_disp_op),
@@ -3046,7 +3136,8 @@ WORD32 ih264d_video_decode(iv_obj_t *dec_hdl, void *pv_api_ip, void *pv_api_op)
                             || (ret == IVD_STREAM_WIDTH_HEIGHT_NOT_SUPPORTED)
                             || (ret == ERROR_UNAVAIL_PICBUF_T)
                             || (ret == ERROR_UNAVAIL_MVBUF_T)
-                            || (ret == ERROR_INV_SPS_PPS_T))
+                            || (ret == ERROR_INV_SPS_PPS_T)
+                            || (ret == IVD_DISP_FRM_ZERO_OP_BUF_SIZE))
             {
                 ps_dec->u4_slice_start_code_found = 0;
                 break;
@@ -3871,27 +3962,15 @@ WORD32 ih264d_get_buf_info(iv_obj_t *dec_hdl, void *pv_api_ip, void *pv_api_op)
     UWORD16 pic_wd, pic_ht;
     ivd_ctl_getbufinfo_op_t *ps_ctl_op =
                     (ivd_ctl_getbufinfo_op_t*)pv_api_op;
+    UWORD32 au4_min_out_buf_size[IVD_VIDDEC_MAX_IO_BUFFERS];
     UNUSED(pv_api_ip);
+
     ps_ctl_op->u4_error_code = 0;
 
     ps_dec = (dec_struct_t *)(dec_hdl->pv_codec_handle);
 
     ps_ctl_op->u4_min_num_in_bufs = MIN_IN_BUFS;
-    if(ps_dec->u1_chroma_format == IV_YUV_420P)
-        ps_ctl_op->u4_min_num_out_bufs = MIN_OUT_BUFS_420;
-    else if(ps_dec->u1_chroma_format == IV_YUV_422ILE)
-        ps_ctl_op->u4_min_num_out_bufs = MIN_OUT_BUFS_422ILE;
-    else if(ps_dec->u1_chroma_format == IV_RGB_565)
-        ps_ctl_op->u4_min_num_out_bufs = MIN_OUT_BUFS_RGB565;
-    else if((ps_dec->u1_chroma_format == IV_YUV_420SP_UV)
-                    || (ps_dec->u1_chroma_format == IV_YUV_420SP_VU))
-        ps_ctl_op->u4_min_num_out_bufs = MIN_OUT_BUFS_420SP;
 
-    else
-    {
-        //Invalid chroma format; Error code may be updated, verify in testing if needed
-        return IV_FAIL;
-    }
 
     ps_ctl_op->u4_num_disp_bufs = 1;
 
@@ -3986,37 +4065,15 @@ WORD32 ih264d_get_buf_info(iv_obj_t *dec_hdl, void *pv_api_ip, void *pv_api_op)
                         ps_ctl_op->u4_num_disp_bufs, 32);
     }
 
-    /*!*/
-    if(ps_dec->u1_chroma_format == IV_YUV_420P)
-    {
-        ps_ctl_op->u4_min_out_buf_size[0] = (pic_wd * pic_ht);
-        ps_ctl_op->u4_min_out_buf_size[1] = (pic_wd * pic_ht)
-                        >> 2;
-        ps_ctl_op->u4_min_out_buf_size[2] = (pic_wd * pic_ht)
-                        >> 2;
-    }
-    else if(ps_dec->u1_chroma_format == IV_YUV_422ILE)
-    {
-        ps_ctl_op->u4_min_out_buf_size[0] = (pic_wd * pic_ht)
-                        * 2;
-        ps_ctl_op->u4_min_out_buf_size[1] =
-                        ps_ctl_op->u4_min_out_buf_size[2] = 0;
-    }
-    else if(ps_dec->u1_chroma_format == IV_RGB_565)
-    {
-        ps_ctl_op->u4_min_out_buf_size[0] = (pic_wd * pic_ht)
-                        * 2;
-        ps_ctl_op->u4_min_out_buf_size[1] =
-                        ps_ctl_op->u4_min_out_buf_size[2] = 0;
-    }
-    else if((ps_dec->u1_chroma_format == IV_YUV_420SP_UV)
-                    || (ps_dec->u1_chroma_format == IV_YUV_420SP_VU))
+    ps_ctl_op->u4_min_num_out_bufs = ih264d_get_outbuf_size(
+                    pic_wd, pic_ht, ps_dec->u1_chroma_format,
+                    &au4_min_out_buf_size[0]);
+
+    for(i = 0; i < ps_ctl_op->u4_min_num_out_bufs; i++)
     {
-        ps_ctl_op->u4_min_out_buf_size[0] = (pic_wd * pic_ht);
-        ps_ctl_op->u4_min_out_buf_size[1] = (pic_wd * pic_ht)
-                        >> 1;
-        ps_ctl_op->u4_min_out_buf_size[2] = 0;
+        ps_ctl_op->u4_min_out_buf_size[i] = au4_min_out_buf_size[i];
     }
+
     ps_dec->u4_num_disp_bufs_requested = ps_ctl_op->u4_num_disp_bufs;
 
     return IV_SUCCESS;
diff --git a/decoder/ih264d_parse_slice.c b/decoder/ih264d_parse_slice.c
index e1ce100..944d580 100644
--- a/decoder/ih264d_parse_slice.c
+++ b/decoder/ih264d_parse_slice.c
@@ -72,6 +72,7 @@
 #include "ih264d_parse_islice.h"
 #define RET_LAST_SKIP  0x80000000
 
+WORD32 check_app_out_buf_size(dec_struct_t *ps_dec);
 /*!
  **************************************************************************
  * \if Function name : ih264d_form_pred_weight_matrix \endif
@@ -181,6 +182,10 @@ WORD32 ih264d_start_of_pic(dec_struct_t *ps_dec,
 
     H264_MUTEX_LOCK(&ps_dec->process_disp_mutex);
 
+    /* check output buffer size given by the application */
+    if(check_app_out_buf_size(ps_dec) != IV_SUCCESS)
+        return IVD_DISP_FRM_ZERO_OP_BUF_SIZE;
+
     ps_prev_poc->i4_pic_order_cnt_lsb = ps_cur_poc->i4_pic_order_cnt_lsb;
     ps_prev_poc->i4_pic_order_cnt_msb = ps_cur_poc->i4_pic_order_cnt_msb;
     ps_prev_poc->i4_delta_pic_order_cnt_bottom =