From 4bc3e63c30c71c6831133ef6bef893bd9fad1fb4 Mon Sep 17 00:00:00 2001 From: Chamarthi Kishore Date: Wed, 27 Nov 2019 13:52:47 +0530 Subject: encoder: Return gracefully from entropy encoding errors Recent error propagation change in encoder introduced a DoS when output buffer is not large enough to hold the frame encoded. This commit fixes the DoS seen. Test: avcenc -c enc.cfg, avcdec -c dec.cfg Test: revert fix for b/144928581 to reduce the output buffer size Test: atest android.media.cts.VideoEncoderTest#testGoogH264FlexMaxMax Test: atest android.media.cts.VideoEncoderTest#testGoogH264Flex1080p Bug:145019703 Change-Id: I3b3408153a2cb8541a87547d51dd8bde25a33d6c --- encoder/ih264e_encode_header.h | 14 +++++++++++ encoder/ih264e_process.c | 53 ++++++++++++------------------------------ 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/encoder/ih264e_encode_header.h b/encoder/ih264e_encode_header.h index 2f7ac91..a028930 100644 --- a/encoder/ih264e_encode_header.h +++ b/encoder/ih264e_encode_header.h @@ -92,6 +92,20 @@ } \ } +/** +****************************************************************************** + * @brief Macro to set active entropy threads to zero and return + * in case of errors +****************************************************************************** + */ +#define RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel) \ + if(ps_entropy->i4_error_code != IH264E_SUCCESS) \ + { \ + DATA_SYNC(); \ + ps_codec->au4_entropy_thread_active[ctxt_sel] = 0; \ + return ps_entropy->i4_error_code; \ + } + /*****************************************************************************/ /* Extern Function Declarations */ /*****************************************************************************/ diff --git a/encoder/ih264e_process.c b/encoder/ih264e_process.c index e5c85a4..5333046 100644 --- a/encoder/ih264e_process.c +++ b/encoder/ih264e_process.c @@ -380,16 +380,11 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc) /* generate sps */ ps_entropy->i4_error_code = ih264e_generate_sps(ps_bitstrm, ps_sps, &ps_codec->s_cfg.s_vui); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); /* generate pps */ ps_entropy->i4_error_code = ih264e_generate_pps(ps_bitstrm, ps_pps, ps_sps); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); + /* reset i4_gen_header */ ps_entropy->i4_gen_header = 0; } @@ -434,20 +429,14 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc) { ps_entropy->i4_error_code = ih264e_generate_sei(ps_bitstrm, &s_sei, u4_insert_per_idr); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); } ps_codec->as_inp_list[ps_codec->i4_poc % MAX_NUM_BFRAMES].u1_sei_ccv_params_present_flag = 0; /* generate slice header */ ps_entropy->i4_error_code = ih264e_generate_slice_header(ps_bitstrm, ps_slice_hdr, ps_pps, ps_sps); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); /* once start of frame / slice is done, you can reset it */ /* it is the responsibility of the caller to set this flag */ ps_entropy->i4_sof = 0; @@ -507,10 +496,7 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc) /* write mb layer */ ps_entropy->i4_error_code = ps_codec->pf_write_mb_syntax_layer [ps_entropy->u1_entropy_coding_mode_flag][i4_slice_type](ps_entropy); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); /* Starting bitstream offset for header in bits */ bitstream_start_offset = GET_NUM_BITS(ps_bitstrm); @@ -553,16 +539,15 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc) { if (*ps_entropy->pi4_mb_skip_run) { - PUT_BITS_UEV(ps_bitstrm, *ps_entropy->pi4_mb_skip_run, ps_entropy->i4_error_code, "mb skip run"); + PUT_BITS_UEV(ps_bitstrm, *ps_entropy->pi4_mb_skip_run, + ps_entropy->i4_error_code, "mb skip run"); *ps_entropy->pi4_mb_skip_run = 0; + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); } } /* put rbsp trailing bits for the previous slice */ - ps_entropy->i4_error_code = ih264e_put_rbsp_trailing_bits(ps_bitstrm); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + ps_entropy->i4_error_code = ih264e_put_rbsp_trailing_bits(ps_bitstrm); + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); } else { @@ -582,10 +567,7 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc) /* generate slice header */ ps_entropy->i4_error_code = ih264e_generate_slice_header( ps_bitstrm, ps_slice_hdr, ps_pps, ps_sps); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); if (CABAC == ps_entropy->u1_entropy_coding_mode_flag) { BITSTREAM_BYTE_ALIGN(ps_bitstrm); @@ -640,14 +622,12 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc) PUT_BITS_UEV(ps_bitstrm, *ps_entropy->pi4_mb_skip_run, ps_entropy->i4_error_code, "mb skip run"); *ps_entropy->pi4_mb_skip_run = 0; + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); } } /* put rbsp trailing bits */ ps_entropy->i4_error_code = ih264e_put_rbsp_trailing_bits(ps_bitstrm); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); } else { @@ -673,10 +653,7 @@ IH264E_ERROR_T ih264e_entropy(process_ctxt_t *ps_proc) { /* add filler nal units */ ps_entropy->i4_error_code = ih264e_add_filler_nal_unit(ps_bitstrm, i4_stuff_bytes); - if(ps_entropy->i4_error_code != IH264E_SUCCESS) - { - return ps_entropy->i4_error_code; - } + RETURN_ENTROPY_IF_ERROR(ps_codec, ps_entropy, ctxt_sel); } } -- cgit v1.2.3