am 696f2291: Fix floating point errors

* commit '696f2291b2f1d4905b7e30290bc4c0daba55ca6c':
  Fix floating point errors

1 files changed, 6 insertions, 2 deletions

diff --git a/src/com/android/calculator2/CalculatorExpressionEvaluator.java b/src/com/android/calculator2/CalculatorExpressionEvaluator.java
index 0f41423..992735c 100644
--- a/src/com/android/calculator2/CalculatorExpressionEvaluator.java
+++ b/src/com/android/calculator2/CalculatorExpressionEvaluator.java
@@ -23,7 +23,8 @@ import org.javia.arity.SyntaxException;
 import org.javia.arity.Util;
 
 public class CalculatorExpressionEvaluator {
-    private static final int MAX_DIGITS = 14;
+    private static final int MAX_DIGITS = 12;
+    private static final int ROUNDING_DIGITS = 1;
 
     private final Symbols mSymbols;
     private final CalculatorExpressionTokenizer mTokenizer;
@@ -65,8 +66,11 @@ public class CalculatorExpressionEvaluator {
             if (Double.isNaN(result)) {
                 callback.onEvaluate(expr, null, mErrorNaN);
             } else {
+                // The arity library uses floating point arithmetic when evaluating the expression
+                // leading to precision errors in the result. The method doubleToString hides these
+                // errors; rounding the result by dropping N digits of precision.
                 callback.onEvaluate(expr, mTokenizer.getLocalizedExpression(
-                        Util.doubleToString(result, MAX_DIGITS, 0)), null);
+                        Util.doubleToString(result, MAX_DIGITS, ROUNDING_DIGITS)), null);
             }
         } catch (SyntaxException e) {
             callback.onEvaluate(expr, null, mErrorSyntax);