Support ignored output in test harness

Filtered out outputs that are not needed for correctness verification
during result validation.

Bug: 63905942
Test: VtsHalNeuralnetworksV1_0TargetTest on Angler
Change-Id: Ic967a821c2a60194c9d9f386601a865f5c409a4f

2 files changed, 18 insertions, 6 deletions

diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index 9c85c8c2cc..db90ac2267 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -33,6 +33,7 @@ hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem
 
 namespace generated_tests {
 using ::android::hardware::neuralnetworks::V1_0::implementation::Event;
+using ::generated_tests::filter;
 using ::generated_tests::for_all;
 using ::generated_tests::for_each;
 using ::generated_tests::resize_accordingly;
@@ -44,6 +45,7 @@ using ::generated_tests::Quant8Operands;
 // Top level driver for models and examples generated by test_generator.py
 // Test driver for those generated from ml/nn/runtime/test/spec
 void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
+             std::function<bool(int)> is_ignored,
              const std::vector<MixedTypedExampleType>& examples) {
     Model model = create_model();
     sp<IPreparedModel> preparedModel;
@@ -166,18 +168,28 @@ void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
         COPY_BACK(uint8_t);
 #undef COPY_BACK
         outputMemory->commit();
+        // Filter out don't cares
+        MixedTyped filtered_golden;
+        MixedTyped filtered_test;
+        filter<float>(golden, &filtered_golden, is_ignored);
+        filter<float>(test, &filtered_test, is_ignored);
+        filter<int32_t>(golden, &filtered_golden, is_ignored);
+        filter<int32_t>(test, &filtered_test, is_ignored);
+        filter<uint8_t>(golden, &filtered_golden, is_ignored);
+        filter<uint8_t>(test, &filtered_test, is_ignored);
+
         // We want "close-enough" results for float
-        for_each<float>(golden, [&test](int index, auto& golden_float) {
-            auto& test_float_operands = std::get<Float32Operands>(test);
+        for_each<float>(filtered_golden, [&filtered_test](int index, auto& golden_float) {
+            auto& test_float_operands = std::get<Float32Operands>(filtered_test);
             auto& test_float = test_float_operands[index];
             for (unsigned int i = 0; i < golden_float.size(); i++) {
                 SCOPED_TRACE(i);
                 EXPECT_NEAR(golden_float[i], test_float[i], 1.e-5);
             }
         });
-
-        EXPECT_EQ(std::get<Int32Operands>(golden), std::get<Int32Operands>(test));
-        EXPECT_EQ(std::get<Quant8Operands>(golden), std::get<Quant8Operands>(test));
+        EXPECT_EQ(std::get<Int32Operands>(filtered_golden), std::get<Int32Operands>(filtered_test));
+        EXPECT_EQ(std::get<Quant8Operands>(filtered_golden),
+                  std::get<Quant8Operands>(filtered_test));
     }
 }
 
diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp
index 0302717767..453e3e57bb 100644
--- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp
+++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp
@@ -34,7 +34,7 @@ namespace functional {
 using ::android::hardware::neuralnetworks::V1_0::implementation::Event;
 using ::generated_tests::MixedTypedExampleType;
 namespace generated_tests {
-extern void Execute(const sp<IDevice>&, std::function<Model(void)>,
+extern void Execute(const sp<IDevice>&, std::function<Model(void)>, std::function<bool(int)>,
                     const std::vector<MixedTypedExampleType>&);
 }