DO NOT MERGE | libvpx: cherry-pick aa1c813 from upstream

CYNGNOS-3235

Description from upstream:
vp9: Fix potential SEGV in decoder_peek_si_internal

decoder_peek_si_internal could potentially read more bytes than
what actually exists in the input buffer. We check for the buffer
size to be at least 8, but we try to read up to 10 bytes in the
worst case. A well crafted file could thus cause a segfault.
Likely change that introduced this bug was:
https://chromium-review.googlesource.com/#/c/70439 (git hash:
7c43fb6)

Bug: 30013856
Change-Id: If556414cb5b82472d5673e045bc185cc57bb9af3
(cherry picked from commit bd57d587c2eb743c61b049add18f9fd72bf78c33)
(cherry picked from commit 909f63f1b23e78d03b5a54612434c7c1f0911980)

1 files changed, 14 insertions, 5 deletions

diff --git a/libvpx/vp9/vp9_dx_iface.c b/libvpx/vp9/vp9_dx_iface.c
index 96ede3c..4eb2fb0 100644
--- a/libvpx/vp9/vp9_dx_iface.c
+++ b/libvpx/vp9/vp9_dx_iface.c
@@ -174,7 +174,7 @@ static vpx_codec_err_t decoder_peek_si_internal(const uint8_t *data,
                                                 vpx_decrypt_cb decrypt_cb,
                                                 void *decrypt_state) {
   int intra_only_flag = 0;
-  uint8_t clear_buffer[9];
+  uint8_t clear_buffer[10];
 
   if (data + data_sz <= data)
     return VPX_CODEC_INVALID_PARAM;
@@ -188,6 +188,11 @@ static vpx_codec_err_t decoder_peek_si_internal(const uint8_t *data,
     data = clear_buffer;
   }
 
+  // A maximum of 6 bits are needed to read the frame marker, profile and
+  // show_existing_frame.
+  if (data_sz < 1)
+    return VPX_CODEC_UNSUP_BITSTREAM;
+
   {
     int show_frame;
     int error_resilient;
@@ -201,15 +206,19 @@ static vpx_codec_err_t decoder_peek_si_internal(const uint8_t *data,
     if (profile >= MAX_PROFILES)
       return VPX_CODEC_UNSUP_BITSTREAM;
 
-    if ((profile >= 2 && data_sz <= 1) || data_sz < 1)
-      return VPX_CODEC_UNSUP_BITSTREAM;
-
     if (vpx_rb_read_bit(&rb)) {  // show an existing frame
+      // If profile is > 2 and show_existing_frame is true, then at least 1 more
+      // byte (6+3=9 bits) is needed.
+      if (profile > 2 && data_sz < 2)
+        return VPX_CODEC_UNSUP_BITSTREAM;
       vpx_rb_read_literal(&rb, 3);  // Frame buffer to show.
       return VPX_CODEC_OK;
     }
 
-    if (data_sz <= 8)
+    // For the rest of the function, a maximum of 9 more bytes are needed
+    // (computed by taking the maximum possible bits needed in each case). Note
+    // that this has to be updated if we read any more bits in this function.
+    if (data_sz < 10)
       return VPX_CODEC_UNSUP_BITSTREAM;
 
     si->is_kf = !vpx_rb_read_bit(&rb);