DO NOT MERGE | libvpx: cherry pick fix to OOB of mv_cost index.HEADreplicant-6.0-0004-transitionreplicant-6.0-0004-rc6replicant-6.0-0004-rc5-transitionreplicant-6.0-0004-rc5replicant-6.0-0004-rc4replicant-6.0-0004-rc3replicant-6.0-0004-rc2replicant-6.0-0004-rc1replicant-6.0-0004cm-13.0

Bug: b/72510002
Test: poc provided in the bug.
Change-Id: I011763cd35b3bac1621e19e4447c442a39d66011
(cherry picked from commit 073990fb32f39c64e4ba9cbd5fb11bfc0c1f0aad)
CVE-2018-9349

1 files changed, 13 insertions, 5 deletions

diff --git a/libvpx/vp8/encoder/mcomp.c b/libvpx/vp8/encoder/mcomp.c
index f848e8f..13a56a7 100644
--- a/libvpx/vp8/encoder/mcomp.c
+++ b/libvpx/vp8/encoder/mcomp.c
@@ -35,16 +35,24 @@ int vp8_mv_bit_cost(int_mv *mv, int_mv *ref, int *mvcost[2], int Weight)
      * NEAREST for subsequent blocks. The "Weight" parameter allows, to a
      * limited extent, for some account to be taken of these factors.
      */
-    return ((mvcost[0][(mv->as_mv.row - ref->as_mv.row) >> 1] + mvcost[1][(mv->as_mv.col - ref->as_mv.col) >> 1]) * Weight) >> 7;
+     const int mv_idx_row =
+         clamp((mv->as_mv.row - ref->as_mv.row) >> 1, 0, MVvals);
+     const int mv_idx_col =
+         clamp((mv->as_mv.col - ref->as_mv.col) >> 1, 0, MVvals);
+     return ((mvcost[0][mv_idx_row] + mvcost[1][mv_idx_col]) * Weight) >> 7;
 }
 
 static int mv_err_cost(int_mv *mv, int_mv *ref, int *mvcost[2], int error_per_bit)
 {
     /* Ignore mv costing if mvcost is NULL */
-    if (mvcost)
-        return ((mvcost[0][(mv->as_mv.row - ref->as_mv.row) >> 1] +
-                 mvcost[1][(mv->as_mv.col - ref->as_mv.col) >> 1])
-                 * error_per_bit + 128) >> 8;
+    if (mvcost) {
+        const int mv_idx_row =
+            clamp((mv->as_mv.row - ref->as_mv.row) >> 1, 0, MVvals);
+        const int mv_idx_col =
+            clamp((mv->as_mv.col - ref->as_mv.col) >> 1, 0, MVvals);
+        return ((mvcost[0][mv_idx_row] + mvcost[1][mv_idx_col]) * error_per_bit +
+                 128) >> 8;
+    }
     return 0;
 }