Notify Errors Appropriately from SoftMPEG2

This is to ensure that the framework realises that an error has
occured and does not call onQueueFilled if the error is fatal.

Bug: 36715268

Change-Id: If02499d306aa9b6df64a7f25d43d1bdf318a10cd
(cherry picked from commit eae3ad0585d17bab12bbd831b58407c122732a1c)
CVE-2017-0759

2 files changed, 38 insertions, 7 deletions

diff --git a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
index e134d38831..ffc79d9b56 100644
--- a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
+++ b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
@@ -68,24 +68,36 @@ SoftMPEG2::SoftMPEG2(
             kProfileLevels, ARRAY_SIZE(kProfileLevels),
             320 /* width */, 240 /* height */, callbacks,
             appData, component),
+      mCodecCtx(NULL),
       mMemRecords(NULL),
       mFlushOutBuffer(NULL),
       mOmxColorFormat(OMX_COLOR_FormatYUV420Planar),
       mIvColorFormat(IV_YUV_420P),
       mNewWidth(mWidth),
       mNewHeight(mHeight),
-      mChangingResolution(false) {
+      mChangingResolution(false),
+      mSignalledError(false) {
     initPorts(kNumBuffers, INPUT_BUF_SIZE, kNumBuffers, CODEC_MIME_TYPE);
 
     // If input dump is enabled, then open create an empty file
     GENERATE_FILE_NAMES();
     CREATE_DUMP_FILE(mInFile);
 
-    CHECK_EQ(initDecoder(), (status_t)OK);
+    if (OK != initDecoder()) {
+        ALOGE("Failed to initialize decoder");
+        notify(OMX_EventError, OMX_ErrorUnsupportedSetting, 0, NULL);
+        mSignalledError = true;
+        return;
+    }
 }
 
 SoftMPEG2::~SoftMPEG2() {
-    CHECK_EQ(deInitDecoder(), (status_t)OK);
+    if (OK != deInitDecoder()) {
+        ALOGE("Failed to deinit decoder");
+        notify(OMX_EventError, OMX_ErrorUnsupportedSetting, 0, NULL);
+        mSignalledError = true;
+        return;
+    }
 }
 
 
@@ -202,6 +214,7 @@ status_t SoftMPEG2::resetDecoder() {
     /* Set number of cores/threads to be used by the codec */
     setNumCores();
 
+    mSignalledError = false;
     return OK;
 }
 
@@ -429,6 +442,7 @@ status_t SoftMPEG2::deInitDecoder() {
 
     mInitNeeded = true;
     mChangingResolution = false;
+    mCodecCtx = NULL;
 
     return OK;
 }
@@ -440,10 +454,13 @@ status_t SoftMPEG2::reInitDecoder() {
 
     ret = initDecoder();
     if (OK != ret) {
-        ALOGE("Create failure");
+        ALOGE("Failed to initialize decoder");
         deInitDecoder();
-        return NO_MEMORY;
+        notify(OMX_EventError, OMX_ErrorUnsupportedSetting, 0, NULL);
+        mSignalledError = true;
+        return ret;
     }
+    mSignalledError = false;
     return OK;
 }
 
@@ -541,6 +558,9 @@ void SoftMPEG2::onPortFlushCompleted(OMX_U32 portIndex) {
 void SoftMPEG2::onQueueFilled(OMX_U32 portIndex) {
     UNUSED(portIndex);
 
+    if (mSignalledError) {
+        return;
+    }
     if (mOutputPortSettingsChange != NONE) {
         return;
     }
@@ -601,7 +621,12 @@ void SoftMPEG2::onQueueFilled(OMX_U32 portIndex) {
             bool portWillReset = false;
             handlePortSettingsChange(&portWillReset, mNewWidth, mNewHeight);
 
-            CHECK_EQ(reInitDecoder(), (status_t)OK);
+            if (OK != reInitDecoder()) {
+                ALOGE("Failed to reinitialize decoder");
+                notify(OMX_EventError, OMX_ErrorUnsupportedSetting, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
             return;
         }
 
@@ -672,7 +697,12 @@ void SoftMPEG2::onQueueFilled(OMX_U32 portIndex) {
                 bool portWillReset = false;
                 handlePortSettingsChange(&portWillReset, s_dec_op.u4_pic_wd, s_dec_op.u4_pic_ht);
 
-                CHECK_EQ(reInitDecoder(), (status_t)OK);
+                if (OK != reInitDecoder()) {
+                    ALOGE("Failed to reinitialize decoder");
+                    notify(OMX_EventError, OMX_ErrorUnsupportedSetting, 0, NULL);
+                    mSignalledError = true;
+                    return;
+                }
 
                 if (setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
                     ivdec_api_function(mCodecCtx, (void *)&s_dec_ip, (void *)&s_dec_op);
diff --git a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
index f48b70b286..e7cbc0eec5 100644
--- a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
+++ b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
@@ -105,6 +105,7 @@ private:
     // codec. So the codec is switching to decode the new resolution.
     bool mChangingResolution;
     bool mFlushNeeded;
+    bool mSignalledError;
     bool mWaitForI;
 
     status_t initDecoder();