Merge "Test execution of large number of random models" am: c35ad48f41 am: 92b961d1b8

Original change: https://android-review.googlesource.com/c/platform/test/mlts/benchmark/+/1327275

Change-Id: Ib1f7ed1bf9b7afcc8bdd84edb71fdbc471cf1c53

5 files changed, 181 insertions, 110 deletions

diff --git a/README.txt b/README.txt
index 61e7d12..99042b9 100644
--- a/README.txt
+++ b/README.txt
@@ -69,4 +69,6 @@ early termination
 * memory-mapped-model-load-stress: runs a series of parallel model compilation with memory mapped
 TFLite models
 
-* model-load-random-stress: test compiling a large set of randomly generated models
\ No newline at end of file
+* model-load-random-stress: test compiling a large set of randomly generated models
+
+* inference-random-stress: test running a large set of randomly generated models
\ No newline at end of file
diff --git a/build_and_run_benchmark.sh b/build_and_run_benchmark.sh
index 2b9a401..7cf7f3b 100755
--- a/build_and_run_benchmark.sh
+++ b/build_and_run_benchmark.sh
@@ -48,6 +48,7 @@ done
 
 MODE="${1:-scoring}"
 INSTALL_NATIVE_TESTS=false
+CRASH_TEST_APP="NeuralNetworksApiCrashTest"
 APP="NeuralNetworksApiBenchmark"
 case "$MODE" in
   scoring)
@@ -61,41 +62,45 @@ case "$MODE" in
     ;;
   parallel-inference-stress)
     CLASS=com.android.nn.crashtest.app.NNParallelCrashResistantInferenceTest
-    APP="NeuralNetworksApiCrashTest"
+    APP="$CRASH_TEST_APP"
     ;;
   parallel-inference-stress-in-process)
     CLASS=com.android.nn.crashtest.app.NNParallelInProcessInferenceTest
-    APP="NeuralNetworksApiCrashTest"
+    APP="$CRASH_TEST_APP"
     ;;
   client-early-termination-stress)
     CLASS=com.android.nn.crashtest.app.NNClientEarlyTerminationTest
-    APP="NeuralNetworksApiCrashTest"
+    APP="$CRASH_TEST_APP"
     ;;
   multi-process-inference-stress)
     CLASS=com.android.nn.crashtest.app.NNMultipleProcessInferenceTest
-    APP="NeuralNetworksApiCrashTest"
+    APP="$CRASH_TEST_APP"
     INSTALL_NATIVE_TESTS=true
     ;;
   multi-process-model-load-stress)
     CLASS=com.android.nn.crashtest.app.NNMultipleProcessModelLoadTest
-    APP="NeuralNetworksApiCrashTest"
+    APP="$CRASH_TEST_APP"
     INSTALL_NATIVE_TESTS=true
     ;;
   memory-mapped-model-load-stress)
     CLASS=com.android.nn.crashtest.app.NNMemoryMappedModelCompilationTest
-    APP="NeuralNetworksApiCrashTest"
+    APP="$CRASH_TEST_APP"
     ;;
   model-load-random-stress)
-    APP="NeuralNetworksApiCrashTest"
+    APP="$CRASH_TEST_APP"
     CLASS=com.android.nn.crashtest.app.NNRandomGraphLoadTest
     ;;
+  inference-random-stress)
+    APP="$CRASH_TEST_APP"
+    CLASS=com.android.nn.crashtest.app.NNRandomGraphExecutionTest
+    ;;
   *)
     echo "Unknown execution mode: $1"
     echo "Known modes: scoring (default), inference-stress, model-loading-stress, " \
       "parallel-inference-stress, parallel-inference-stress-in-process, " \
       "client-early-termination-stress, multi-process-inference-stress, " \
       "multi-process-model-load-stress memory-mapped-model-load-stress, " \
-      "model-load-random-stress"
+      "model-load-random-stress  inference-random-stress"
     exit 1
     ;;
 esac
diff --git a/src/com/android/nn/crashtest/app/NNRandomGraphExecutionTest.java b/src/com/android/nn/crashtest/app/NNRandomGraphExecutionTest.java
new file mode 100644
index 0000000..e05cd99
--- /dev/null
+++ b/src/com/android/nn/crashtest/app/NNRandomGraphExecutionTest.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.nn.crashtest.app;
+
+import com.android.nn.crashtest.app.NNRandomGraphTest;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.time.Duration;
+
+@RunWith(Parameterized.class)
+public class NNRandomGraphExecutionTest extends NNRandomGraphTest {
+    public NNRandomGraphExecutionTest(int modelCount, int graphSize, int dimensionRange,
+            Duration duration, String acceleratorName) {
+        super(modelCount, graphSize, dimensionRange, duration,
+                acceleratorName, /*runModelCompilationOnly=*/false);
+    }
+}
\ No newline at end of file
diff --git a/src/com/android/nn/crashtest/app/NNRandomGraphLoadTest.java b/src/com/android/nn/crashtest/app/NNRandomGraphLoadTest.java
index 80de3ce..34962f4 100644
--- a/src/com/android/nn/crashtest/app/NNRandomGraphLoadTest.java
+++ b/src/com/android/nn/crashtest/app/NNRandomGraphLoadTest.java
@@ -16,114 +16,16 @@
 
 package com.android.nn.crashtest.app;
 
-
-import static com.android.nn.crashtest.app.CrashTestStatus.TestResult.SUCCESS;
-
-import android.content.Intent;
-import android.test.ActivityInstrumentationTestCase2;
-import android.test.UiThreadTest;
-import android.test.suitebuilder.annotation.LargeTest;
-
-import androidx.test.InstrumentationRegistry;
-
-import com.android.nn.benchmark.app.BenchmarkTestBase;
-import com.android.nn.benchmark.core.NnApiDelegationFailure;
-import com.android.nn.benchmark.core.TestModels;
-import com.android.nn.crashtest.core.test.RandomGraphTest;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
 
 import java.time.Duration;
-import java.util.Arrays;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
 
 @RunWith(Parameterized.class)
-public class NNRandomGraphLoadTest
-        extends ActivityInstrumentationTestCase2<NNRandomGraphTestActivity>
-        implements AcceleratorSpecificTestSupport {
-    private static final String TAG = "NN_RAND_MODEL";
-
-    protected final String mAcceleratorName;
-    private final int mModelCount;
-    private final int mGraphSize;
-    private final Duration mDuration;
-    private final int mDimensionRange;
-
-    private final static int SMALL_MODEL_SIZE = 10;
-    private final static int LARGE_MODEL_SIZE = 600;
-    private final static int NARROW_DIMENSIONS_RANGE = 10;
-    private final static int WIDE_DIMENSIONS_RANGE = 1000;
-    private final static Duration MAX_TEST_DURATION = Duration.ofMinutes(15);
-    private final static int NUMBER_OF_MODELS = 600;
-
-    @Parameters(name =
-            "{0} models of size {1} and dimensions range {2} for max duration of {3} on accelerator"
-                    + " {4}")
-    public static Iterable<Object[]> testConfiguration() {
-        return AcceleratorSpecificTestSupport.perAcceleratorTestConfig(
-                Arrays.asList(
-                        new Object[]{NUMBER_OF_MODELS, SMALL_MODEL_SIZE, WIDE_DIMENSIONS_RANGE,
-                                MAX_TEST_DURATION},
-                        new Object[]{NUMBER_OF_MODELS, LARGE_MODEL_SIZE, NARROW_DIMENSIONS_RANGE,
-                                MAX_TEST_DURATION}));
-    }
-
-    @Rule
-    public TestName mTestName = new TestName();
-
+public class NNRandomGraphLoadTest extends NNRandomGraphTest {
     public NNRandomGraphLoadTest(int modelCount, int graphSize, int dimensionRange,
             Duration duration, String acceleratorName) {
-        super(NNRandomGraphTestActivity.class);
-        mModelCount = modelCount;
-        mGraphSize = graphSize;
-        mDuration = duration;
-        mAcceleratorName = acceleratorName;
-        mDimensionRange = dimensionRange;
-    }
-
-    @Before
-    @Override
-    public void setUp() {
-        injectInstrumentation(InstrumentationRegistry.getInstrumentation());
-        BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 60);
-        setActivityIntent(getLoadModelsOfSizeAndRangeForMaxTimeIntent(mGraphSize, mDimensionRange,
-                mModelCount, mAcceleratorName, mDuration, mTestName.getMethodName()));
-    }
-
-    protected Optional<TestModels.TestModelEntry> findModelForLivenessTest()
-            throws NnApiDelegationFailure {
-        return findTestModelRunningOnAccelerator(
-                getInstrumentation().getTargetContext(), mAcceleratorName);
-    }
-
-    @Test
-    @LargeTest
-    @UiThreadTest
-    public void testDriverDoesNotFailWithParallelWorkload()
-            throws ExecutionException, InterruptedException, NnApiDelegationFailure {
-        final NNRandomGraphTestActivity activity = getActivity();
-
-        assertEquals(SUCCESS, activity.testResult());
-    }
-
-    /**
-     * @return the intent to use to initialise the RunModelsInMultipleProcesses test class
-     */
-    protected Intent getLoadModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange,
-            int modelsCount, String deviceName, Duration maxTestDuration, String testName) {
-        Intent result = new Intent();
-        RandomGraphTest
-                .intentInitializer(graphSize, dimensionsRange, modelsCount,
-                        RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS,
-                        /*compilation only=*/true, deviceName, maxTestDuration.toMillis(), testName)
-                .addIntentParams(result);
-        return result;
+        super(modelCount, graphSize, dimensionRange, duration,
+                acceleratorName, /*runModelCompilationOnly=*/true);
     }
 }
\ No newline at end of file
diff --git a/src/com/android/nn/crashtest/app/NNRandomGraphTest.java b/src/com/android/nn/crashtest/app/NNRandomGraphTest.java
new file mode 100644
index 0000000..86e3b0f
--- /dev/null
+++ b/src/com/android/nn/crashtest/app/NNRandomGraphTest.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.nn.crashtest.app;
+
+import static com.android.nn.crashtest.app.CrashTestStatus.TestResult.SUCCESS;
+
+import android.content.Intent;
+import android.test.ActivityInstrumentationTestCase2;
+import android.test.UiThreadTest;
+import android.test.suitebuilder.annotation.LargeTest;
+
+import androidx.test.InstrumentationRegistry;
+
+import com.android.nn.benchmark.app.BenchmarkTestBase;
+import com.android.nn.benchmark.core.NnApiDelegationFailure;
+import com.android.nn.benchmark.core.TestModels;
+import com.android.nn.crashtest.core.test.RandomGraphTest;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+@RunWith(Parameterized.class)
+public abstract class NNRandomGraphTest
+        extends ActivityInstrumentationTestCase2<NNRandomGraphTestActivity>
+        implements AcceleratorSpecificTestSupport {
+    private static final String TAG = "NN_RAND_MODEL";
+
+    protected final String mAcceleratorName;
+    private final int mModelCount;
+    private final int mGraphSize;
+    private final Duration mDuration;
+    private final int mDimensionRange;
+
+    private final static int SMALL_MODEL_SIZE = 10;
+    private final static int LARGE_MODEL_SIZE = 600;
+    private final static int NARROW_DIMENSIONS_RANGE = 10;
+    private final static int WIDE_DIMENSIONS_RANGE = 1000;
+    private final static Duration MAX_TEST_DURATION = Duration.ofMinutes(15);
+    private final static int NUMBER_OF_MODELS = 600;
+    private final boolean mRunModelCompilationOnly;
+
+    @Parameters(name = "{0} models of size {1} and dimensions range {2} for max duration of {3} "
+            + "on accelerator {4}")
+    public static Iterable<Object[]> testConfiguration() {
+        return AcceleratorSpecificTestSupport.perAcceleratorTestConfig(
+                Arrays.asList(
+                        new Object[]{NUMBER_OF_MODELS, SMALL_MODEL_SIZE, WIDE_DIMENSIONS_RANGE,
+                                MAX_TEST_DURATION},
+                        new Object[]{NUMBER_OF_MODELS, LARGE_MODEL_SIZE, NARROW_DIMENSIONS_RANGE,
+                                MAX_TEST_DURATION}));
+    }
+
+    @Rule
+    public TestName mTestName = new TestName();
+
+    public NNRandomGraphTest(int modelCount, int graphSize, int dimensionRange,
+            Duration duration, String acceleratorName, boolean runModelCompilationOnly) {
+        super(NNRandomGraphTestActivity.class);
+        mModelCount = modelCount;
+        mGraphSize = graphSize;
+        mDuration = duration;
+        mAcceleratorName = acceleratorName;
+        mDimensionRange = dimensionRange;
+        mRunModelCompilationOnly = runModelCompilationOnly;
+    }
+
+    @Before
+    @Override
+    public void setUp() {
+        injectInstrumentation(InstrumentationRegistry.getInstrumentation());
+        BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 60);
+        setActivityIntent(getTestModelsOfSizeAndRangeForMaxTimeIntent(mGraphSize, mDimensionRange,
+                mModelCount, mAcceleratorName, mDuration, mTestName.getMethodName()));
+    }
+
+    protected Optional<TestModels.TestModelEntry> findModelForLivenessTest()
+            throws NnApiDelegationFailure {
+        return findTestModelRunningOnAccelerator(
+                getInstrumentation().getTargetContext(), mAcceleratorName);
+    }
+
+    @Test
+    @LargeTest
+    @UiThreadTest
+    public void testDriverDoesNotFailWithParallelWorkload()
+            throws ExecutionException, InterruptedException, NnApiDelegationFailure {
+        final NNRandomGraphTestActivity activity = getActivity();
+
+        assertEquals(SUCCESS, activity.testResult());
+    }
+
+    /**
+     * @return the intent to use to initialise the RandomGraphTest test class
+     */
+    protected Intent getTestModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange,
+            int modelsCount, String deviceName, Duration maxTestDuration, String testName) {
+        Intent result = new Intent();
+        RandomGraphTest
+                .intentInitializer(graphSize, dimensionsRange, modelsCount,
+                        RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS,
+                        mRunModelCompilationOnly, deviceName, maxTestDuration.toMillis(), testName)
+                .addIntentParams(result);
+        return result;
+    }
+}
\ No newline at end of file