Added error check for output buffer size.

The output buffer size given by the application, needs to be checked
in every process call. This is required in the case of resolution
change.

Bug: 36006815
Test: avcdec -i poc.bin
Change-Id: I16a92cdad23eb7b1e12c1a67c1b2599204f29249
(cherry picked from commit 3f6c941de5cd959072fa046c9d6cb26fa0f01dc6)

2 files changed, 106 insertions, 44 deletions

diff --git a/decoder/ih264d_api.c b/decoder/ih264d_api.c
index 2cde456..7c3f750 100644
--- a/decoder/ih264d_api.c
+++ b/decoder/ih264d_api.c
@@ -1623,6 +1623,89 @@ UWORD32 ih264d_map_error(UWORD32 i4_err_status)
 
 }
 
+UWORD32 ih264d_get_outbuf_size(WORD32 pic_wd,
+                               UWORD32 pic_ht,
+                               UWORD8 u1_chroma_format,
+                               UWORD32 *p_buf_size)
+{
+    UWORD32 u4_min_num_out_bufs = 0;
+
+    if(u1_chroma_format == IV_YUV_420P)
+        u4_min_num_out_bufs = MIN_OUT_BUFS_420;
+    else if(u1_chroma_format == IV_YUV_422ILE)
+        u4_min_num_out_bufs = MIN_OUT_BUFS_422ILE;
+    else if(u1_chroma_format == IV_RGB_565)
+        u4_min_num_out_bufs = MIN_OUT_BUFS_RGB565;
+    else if((u1_chroma_format == IV_YUV_420SP_UV)
+                    || (u1_chroma_format == IV_YUV_420SP_VU))
+        u4_min_num_out_bufs = MIN_OUT_BUFS_420SP;
+
+    if(u1_chroma_format == IV_YUV_420P)
+    {
+        p_buf_size[0] = (pic_wd * pic_ht);
+        p_buf_size[1] = (pic_wd * pic_ht) >> 2;
+        p_buf_size[2] = (pic_wd * pic_ht) >> 2;
+    }
+    else if(u1_chroma_format == IV_YUV_422ILE)
+    {
+        p_buf_size[0] = (pic_wd * pic_ht) * 2;
+        p_buf_size[1] = p_buf_size[2] = 0;
+    }
+    else if(u1_chroma_format == IV_RGB_565)
+    {
+        p_buf_size[0] = (pic_wd * pic_ht) * 2;
+        p_buf_size[1] = p_buf_size[2] = 0;
+    }
+    else if((u1_chroma_format == IV_YUV_420SP_UV)
+                    || (u1_chroma_format == IV_YUV_420SP_VU))
+    {
+        p_buf_size[0] = (pic_wd * pic_ht);
+        p_buf_size[1] = (pic_wd * pic_ht) >> 1;
+        p_buf_size[2] = 0;
+    }
+
+    return u4_min_num_out_bufs;
+}
+
+WORD32 check_app_out_buf_size(dec_struct_t *ps_dec)
+{
+    UWORD32 au4_min_out_buf_size[IVD_VIDDEC_MAX_IO_BUFFERS];
+    UWORD32 u4_min_num_out_bufs, i;
+    UWORD32 pic_wd, pic_ht;
+
+    if(0 == ps_dec->u4_share_disp_buf)
+    {
+        pic_wd = ps_dec->u2_disp_width;
+        pic_ht = ps_dec->u2_disp_height;
+
+    }
+    else
+    {
+        /* In case of shared mode, do not check validity of ps_dec->ps_out_buffer */
+        return (IV_SUCCESS);
+    }
+
+    if(ps_dec->u4_app_disp_width > pic_wd)
+        pic_wd = ps_dec->u4_app_disp_width;
+
+    u4_min_num_out_bufs = ih264d_get_outbuf_size(pic_wd, pic_ht,
+                                                 ps_dec->u1_chroma_format,
+                                                 &au4_min_out_buf_size[0]);
+
+    if(ps_dec->ps_out_buffer->u4_num_bufs < u4_min_num_out_bufs)
+        return IV_FAIL;
+
+    for(i = 0; i < u4_min_num_out_bufs; i++)
+    {
+        if(ps_dec->ps_out_buffer->u4_min_out_buf_size[i]
+                        < au4_min_out_buf_size[i])
+            return (IV_FAIL);
+    }
+
+    return (IV_SUCCESS);
+}
+
+
 /*****************************************************************************/
 /*                                                                           */
 /*  Function Name :  ih264d_video_decode                                     */
@@ -1865,6 +1948,13 @@ WORD32 ih264d_video_decode(iv_obj_t *dec_hdl, void *pv_api_ip, void *pv_api_op)
                                       &(ps_dec->s_disp_op));
         if(0 == ps_dec->s_disp_op.u4_error_code)
         {
+            /* check output buffer size given by the application */
+            if(check_app_out_buf_size(ps_dec) != IV_SUCCESS)
+            {
+                ps_dec_op->u4_error_code= IVD_DISP_FRM_ZERO_OP_BUF_SIZE;
+                return (IV_FAIL);
+            }
+
             ps_dec->u4_fmt_conv_cur_row = 0;
             ps_dec->u4_fmt_conv_num_rows = ps_dec->s_disp_frame_info.u4_y_ht;
             ih264d_format_convert(ps_dec, &(ps_dec->s_disp_op),
@@ -2094,7 +2184,8 @@ WORD32 ih264d_video_decode(iv_obj_t *dec_hdl, void *pv_api_ip, void *pv_api_op)
                             || (ret == IVD_MEM_ALLOC_FAILED)
                             || (ret == ERROR_UNAVAIL_PICBUF_T)
                             || (ret == ERROR_UNAVAIL_MVBUF_T)
-                            || (ret == ERROR_INV_SPS_PPS_T))
+                            || (ret == ERROR_INV_SPS_PPS_T)
+                            || (ret == IVD_DISP_FRM_ZERO_OP_BUF_SIZE))
             {
                 ps_dec->u4_slice_start_code_found = 0;
                 break;
@@ -2843,27 +2934,15 @@ WORD32 ih264d_get_buf_info(iv_obj_t *dec_hdl, void *pv_api_ip, void *pv_api_op)
     UWORD16 pic_wd, pic_ht;
     ivd_ctl_getbufinfo_op_t *ps_ctl_op =
                     (ivd_ctl_getbufinfo_op_t*)pv_api_op;
+    UWORD32 au4_min_out_buf_size[IVD_VIDDEC_MAX_IO_BUFFERS];
     UNUSED(pv_api_ip);
+
     ps_ctl_op->u4_error_code = 0;
 
     ps_dec = (dec_struct_t *)(dec_hdl->pv_codec_handle);
 
     ps_ctl_op->u4_min_num_in_bufs = MIN_IN_BUFS;
-    if(ps_dec->u1_chroma_format == IV_YUV_420P)
-        ps_ctl_op->u4_min_num_out_bufs = MIN_OUT_BUFS_420;
-    else if(ps_dec->u1_chroma_format == IV_YUV_422ILE)
-        ps_ctl_op->u4_min_num_out_bufs = MIN_OUT_BUFS_422ILE;
-    else if(ps_dec->u1_chroma_format == IV_RGB_565)
-        ps_ctl_op->u4_min_num_out_bufs = MIN_OUT_BUFS_RGB565;
-    else if((ps_dec->u1_chroma_format == IV_YUV_420SP_UV)
-                    || (ps_dec->u1_chroma_format == IV_YUV_420SP_VU))
-        ps_ctl_op->u4_min_num_out_bufs = MIN_OUT_BUFS_420SP;
 
-    else
-    {
-        //Invalid chroma format; Error code may be updated, verify in testing if needed
-        return IV_FAIL;
-    }
 
     ps_ctl_op->u4_num_disp_bufs = 1;
 
@@ -2930,37 +3009,15 @@ WORD32 ih264d_get_buf_info(iv_obj_t *dec_hdl, void *pv_api_ip, void *pv_api_op)
                         ps_ctl_op->u4_num_disp_bufs, 32);
     }
 
-    /*!*/
-    if(ps_dec->u1_chroma_format == IV_YUV_420P)
-    {
-        ps_ctl_op->u4_min_out_buf_size[0] = (pic_wd * pic_ht);
-        ps_ctl_op->u4_min_out_buf_size[1] = (pic_wd * pic_ht)
-                        >> 2;
-        ps_ctl_op->u4_min_out_buf_size[2] = (pic_wd * pic_ht)
-                        >> 2;
-    }
-    else if(ps_dec->u1_chroma_format == IV_YUV_422ILE)
-    {
-        ps_ctl_op->u4_min_out_buf_size[0] = (pic_wd * pic_ht)
-                        * 2;
-        ps_ctl_op->u4_min_out_buf_size[1] =
-                        ps_ctl_op->u4_min_out_buf_size[2] = 0;
-    }
-    else if(ps_dec->u1_chroma_format == IV_RGB_565)
-    {
-        ps_ctl_op->u4_min_out_buf_size[0] = (pic_wd * pic_ht)
-                        * 2;
-        ps_ctl_op->u4_min_out_buf_size[1] =
-                        ps_ctl_op->u4_min_out_buf_size[2] = 0;
-    }
-    else if((ps_dec->u1_chroma_format == IV_YUV_420SP_UV)
-                    || (ps_dec->u1_chroma_format == IV_YUV_420SP_VU))
+    ps_ctl_op->u4_min_num_out_bufs = ih264d_get_outbuf_size(
+                    pic_wd, pic_ht, ps_dec->u1_chroma_format,
+                    &au4_min_out_buf_size[0]);
+
+    for(i = 0; i < ps_ctl_op->u4_min_num_out_bufs; i++)
     {
-        ps_ctl_op->u4_min_out_buf_size[0] = (pic_wd * pic_ht);
-        ps_ctl_op->u4_min_out_buf_size[1] = (pic_wd * pic_ht)
-                        >> 1;
-        ps_ctl_op->u4_min_out_buf_size[2] = 0;
+        ps_ctl_op->u4_min_out_buf_size[i] = au4_min_out_buf_size[i];
     }
+
     ps_dec->u4_num_disp_bufs_requested = ps_ctl_op->u4_num_disp_bufs;
 
     return IV_SUCCESS;
diff --git a/decoder/ih264d_parse_slice.c b/decoder/ih264d_parse_slice.c
index fad2dff..ab9f3f6 100644
--- a/decoder/ih264d_parse_slice.c
+++ b/decoder/ih264d_parse_slice.c
@@ -72,6 +72,7 @@
 #include "ih264d_parse_islice.h"
 #define RET_LAST_SKIP  0x80000000
 
+WORD32 check_app_out_buf_size(dec_struct_t *ps_dec);
 /*!
  **************************************************************************
  * \if Function name : ih264d_form_pred_weight_matrix \endif
@@ -181,6 +182,10 @@ WORD32 ih264d_start_of_pic(dec_struct_t *ps_dec,
 
     H264_MUTEX_LOCK(&ps_dec->process_disp_mutex);
 
+    /* check output buffer size given by the application */
+    if(check_app_out_buf_size(ps_dec) != IV_SUCCESS)
+        return IVD_DISP_FRM_ZERO_OP_BUF_SIZE;
+
     ps_prev_poc->i4_pic_order_cnt_lsb = ps_cur_poc->i4_pic_order_cnt_lsb;
     ps_prev_poc->i4_pic_order_cnt_msb = ps_cur_poc->i4_pic_order_cnt_msb;
     ps_prev_poc->i4_delta_pic_order_cnt_bottom =