encoder: Return gracefully from entropy encoding errors

Recent error propagation change in encoder introduced a DoS when
output buffer is not large enough to hold the frame encoded.
This commit fixes the DoS seen.

Test: avcenc -c enc.cfg, avcdec -c dec.cfg
Test: revert fix for b/144928581 to reduce the output buffer size
Test: atest android.media.cts.VideoEncoderTest#testGoogH264FlexMaxMax
Test: atest android.media.cts.VideoEncoderTest#testGoogH264Flex1080p

Bug:145019703

Change-Id: I3b3408153a2cb8541a87547d51dd8bde25a33d6c

2 files changed, 29 insertions, 38 deletions

diff --git a/encoder/ih264e_encode_header.h b/encoder/ih264e_encode_header.h
index 2f7ac91..a028930 100644
--- a/encoder/ih264e_encode_header.h
+++ b/encoder/ih264e_encode_header.h
@@ -92,6 +92,20 @@
             }                                                                \
         }
 
+/**
+******************************************************************************
+ *  @brief   Macro to set active entropy threads to zero and return
+ *           in case of errors
+******************************************************************************
+ */
+#define RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel)              \
+        if(ps_entropy->i4_error_code != IH264E_SUCCESS)                      \
+        {                                                                    \
+            DATA_SYNC();                                                     \
+            ps_codec->au4_entropy_thread_active[ctxt_sel] = 0;               \
+            return ps_entropy->i4_error_code;                                \
+        }
+
 /*****************************************************************************/
 /* Extern Function Declarations                                              */
 /*****************************************************************************/
diff --git a/encoder/ih264e_process.c b/encoder/ih264e_process.c
index e5c85a4..5333046 100644
--- a/encoder/ih264e_process.c
+++ b/encoder/ih264e_process.c
@@ -380,16 +380,11 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc)
             /* generate sps */
             ps_entropy->i4_error_code = ih264e_generate_sps(ps_bitstrm, ps_sps,
                                                              &ps_codec->s_cfg.s_vui);
-            if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-            {
-                return ps_entropy->i4_error_code;
-            }
+            RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
             /* generate pps */
             ps_entropy->i4_error_code = ih264e_generate_pps(ps_bitstrm, ps_pps, ps_sps);
-            if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-            {
-                return ps_entropy->i4_error_code;
-            }
+            RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
+
             /* reset i4_gen_header */
             ps_entropy->i4_gen_header = 0;
         }
@@ -434,20 +429,14 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc)
         {
             ps_entropy->i4_error_code =
                     ih264e_generate_sei(ps_bitstrm, &s_sei, u4_insert_per_idr);
-            if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-            {
-                return ps_entropy->i4_error_code;
-            }
+            RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
         }
         ps_codec->as_inp_list[ps_codec->i4_poc % MAX_NUM_BFRAMES].u1_sei_ccv_params_present_flag = 0;
 
         /* generate slice header */
         ps_entropy->i4_error_code = ih264e_generate_slice_header(ps_bitstrm, ps_slice_hdr,
                                                                   ps_pps, ps_sps);
-        if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-        {
-            return ps_entropy->i4_error_code;
-        }
+        RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
         /* once start of frame / slice is done, you can reset it */
         /* it is the responsibility of the caller to set this flag */
         ps_entropy->i4_sof = 0;
@@ -507,10 +496,7 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc)
         /* write mb layer */
         ps_entropy->i4_error_code = ps_codec->pf_write_mb_syntax_layer
                         [ps_entropy->u1_entropy_coding_mode_flag][i4_slice_type](ps_entropy);
-        if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-        {
-            return ps_entropy->i4_error_code;
-        }
+        RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
 
         /* Starting bitstream offset for header in bits */
         bitstream_start_offset = GET_NUM_BITS(ps_bitstrm);
@@ -553,16 +539,15 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc)
                         {
                             if (*ps_entropy->pi4_mb_skip_run)
                             {
-                            PUT_BITS_UEV(ps_bitstrm, *ps_entropy->pi4_mb_skip_run, ps_entropy->i4_error_code, "mb skip run");
+                                PUT_BITS_UEV(ps_bitstrm, *ps_entropy->pi4_mb_skip_run,
+                                            ps_entropy->i4_error_code, "mb skip run");
                                 *ps_entropy->pi4_mb_skip_run = 0;
+                                RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
                             }
                         }
                         /* put rbsp trailing bits for the previous slice */
-                                ps_entropy->i4_error_code = ih264e_put_rbsp_trailing_bits(ps_bitstrm);
-                                if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-                                {
-                                    return ps_entropy->i4_error_code;
-                                }
+                        ps_entropy->i4_error_code = ih264e_put_rbsp_trailing_bits(ps_bitstrm);
+                        RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
                     }
                     else
                     {
@@ -582,10 +567,7 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc)
                     /* generate slice header */
                     ps_entropy->i4_error_code = ih264e_generate_slice_header(
                                     ps_bitstrm, ps_slice_hdr, ps_pps, ps_sps);
-                    if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-                    {
-                        return ps_entropy->i4_error_code;
-                    }
+                    RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
                     if (CABAC == ps_entropy->u1_entropy_coding_mode_flag)
                     {
                         BITSTREAM_BYTE_ALIGN(ps_bitstrm);
@@ -640,14 +622,12 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc)
                     PUT_BITS_UEV(ps_bitstrm, *ps_entropy->pi4_mb_skip_run,
                                  ps_entropy->i4_error_code, "mb skip run");
                     *ps_entropy->pi4_mb_skip_run = 0;
+                    RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
                 }
             }
             /* put rbsp trailing bits */
              ps_entropy->i4_error_code = ih264e_put_rbsp_trailing_bits(ps_bitstrm);
-             if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-             {
-                 return ps_entropy->i4_error_code;
-             }
+             RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
         }
         else
         {
@@ -673,10 +653,7 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc)
             {
                 /* add filler nal units */
                  ps_entropy->i4_error_code = ih264e_add_filler_nal_unit(ps_bitstrm, i4_stuff_bytes);
-                 if(ps_entropy->i4_error_code != IH264E_SUCCESS)
-                 {
-                     return ps_entropy->i4_error_code;
-                 }
+                 RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel);
             }
         }