Add more debugging to the DigestTest

This test sometimes fails, but we don't know what the coefficient of
variation is on the test results. Add this to the debug output so we can
see if there is just some wild swing during testing.

Also add a few rounds of warm-up as well.

Bug: 24011092
Change-Id: Ic58f106f68eb93976e3f030e2f23e0156fe84be8
(cherry picked from commit 5fe1cd001f38fba460ac0ce5c15b85250e400f25)

2 files changed, 103 insertions, 12 deletions

diff --git a/luni/src/test/java/com/android/org/bouncycastle/crypto/digests/DigestTest.java b/luni/src/test/java/com/android/org/bouncycastle/crypto/digests/DigestTest.java
index 870821416..fce8507d2 100644
--- a/luni/src/test/java/com/android/org/bouncycastle/crypto/digests/DigestTest.java
+++ b/luni/src/test/java/com/android/org/bouncycastle/crypto/digests/DigestTest.java
@@ -19,6 +19,7 @@ package com.android.org.bouncycastle.crypto.digests;
 import junit.framework.TestCase;
 import com.android.org.bouncycastle.crypto.Digest;
 import com.android.org.bouncycastle.crypto.ExtendedDigest;
+import tests.util.SummaryStatistics;
 
 /**
  * Implements unit tests for our JNI wrapper around OpenSSL. We use the
@@ -36,6 +37,7 @@ public class DigestTest extends TestCase {
      * @param newDigest The new digest implementation, provided by OpenSSL
      */
     public void doTestMessageDigest(Digest oldDigest, Digest newDigest) {
+        final int WARMUP = 10;
         final int ITERATIONS = 100;
 
         byte[] data = new byte[1024];
@@ -54,27 +56,31 @@ public class DigestTest extends TestCase {
             data[i] = (byte)i;
         }
 
-        long oldTime = 0;
-        long newTime = 0;
+        SummaryStatistics oldTime = new SummaryStatistics();
+        SummaryStatistics newTime = new SummaryStatistics();
 
-        for (int j = 0; j < ITERATIONS; j++) {
-            long t0 = System.currentTimeMillis();
+        for (int j = 0; j < ITERATIONS + WARMUP; j++) {
+            long t0 = System.nanoTime();
             for (int i = 0; i < 4; i++) {
                 oldDigest.update(data, 0, data.length);
             }
             int oldLength = oldDigest.doFinal(oldHash, 0);
-            long t1 = System.currentTimeMillis();
+            long t1 = System.nanoTime();
 
-            oldTime = oldTime + (t1 - t0);
+            if (j >= WARMUP) {
+                oldTime.add(t1 - t0);
+            }
 
-            long t2 = System.currentTimeMillis();
+            long t2 = System.nanoTime();
             for (int i = 0; i < 4; i++) {
                 newDigest.update(data, 0, data.length);
             }
             int newLength = newDigest.doFinal(newHash, 0);
-            long t3 = System.currentTimeMillis();
+            long t3 = System.nanoTime();
 
-            newTime = newTime + (t3 - t2);
+            if (j >= WARMUP) {
+              newTime.add(t3 - t2);
+            }
 
             assertEquals("Hash sizes must be equal", oldLength, newLength);
 
@@ -83,10 +89,13 @@ public class DigestTest extends TestCase {
             }
         }
 
-        System.out.println("Time for " + ITERATIONS + " x old hash processing: " + oldTime + " ms");
-        System.out.println("Time for " + ITERATIONS + " x new hash processing: " + newTime + " ms");
+        System.out.println("Time for " + ITERATIONS + " x old hash processing: "
+                + oldTime.toString());
+        System.out.println("Time for " + ITERATIONS + " x new hash processing: "
+                + newTime.toString());
 
-        assertTrue("New hash should be faster", newTime < oldTime);
+        assertTrue("New hash should be faster:\nold=" + oldTime.toString() + "\nnew="
+                + newTime.toString(), newTime.mean() < oldTime.mean());
     }
 
     /**
diff --git a/support/src/test/java/tests/util/SummaryStatistics.java b/support/src/test/java/tests/util/SummaryStatistics.java
new file mode 100644
index 000000000..4ce0a0409
--- /dev/null
+++ b/support/src/test/java/tests/util/SummaryStatistics.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tests.util;
+
+public class SummaryStatistics {
+  /** The number of values seen. */
+  private int numValues;
+
+  /** Sum of the values. */
+  private double sum;
+
+  /** Sum of the squares of the values added. */
+  private double squaresSum;
+
+  /** The previously added value. */
+  private double lastValue;
+
+  public SummaryStatistics() {
+  }
+
+  private double square(double value) {
+    return value * value;
+  }
+
+  /** Add a new value to the values seen. */
+  public void add(double value) {
+    sum += value - lastValue;
+    squaresSum += square(value) - square(lastValue);
+    numValues++;
+    lastValue = value;
+  }
+
+  /** Mean of the values seen. */
+  public double mean() {
+    return sum / numValues;
+  }
+
+  /** Variance of the values seen. */
+  public double var() {
+    return (squaresSum / numValues) - square(mean());
+  }
+
+  /** Standard deviation of the values seen. */
+  public double stddev() {
+    return Math.sqrt(var());
+  }
+
+  /** Coefficient of variation of the values seen. */
+  public double coeffVar() {
+    return stddev() / mean();
+  }
+
+  public String toString() {
+    StringBuilder sb = new StringBuilder("SummaryStatistics{");
+    sb.append("n=");
+    sb.append(numValues);
+    sb.append(",mean=");
+    sb.append(mean());
+    sb.append(",var=");
+    sb.append(var());
+    sb.append(",stddev=");
+    sb.append(stddev());
+    sb.append(",coeffVar=");
+    sb.append(coeffVar());
+    sb.append('}');
+    return sb.toString();
+  }
+}