Always call PyNumber_Index when casting from Python to a C++ integral type, also pre-3.8 (#2801)

* Always call PyNumber_Index when casting from Python to a C++ integral type, also pre-3.8

* Fixed on PyPy

* Simplify use of PyNumber_Index, following @rwgk's idea, and ignore warnings in >=3.8

* Reproduce mismatch between pre-3.8 and post-3.8 behavior on __index__ throwing TypeError

* Fix tests on 3.6 <= Python < 3.8

* No, I don't have an uninitialized variable

* Fix use of __index__ on Python 2

* Make types in test_int_convert more ~boring~ descriptive

2 files changed, 60 insertions, 30 deletions

diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index 009bf8f..0caccdb 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -1040,14 +1040,31 @@ public:
                 return false;
         } else if (PyFloat_Check(src.ptr())) {
             return false;
-        } else if (!convert && !index_check(src.ptr()) && !PYBIND11_LONG_CHECK(src.ptr())) {
+        } else if (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr())) {
             return false;
-        } else if (std::is_unsigned<py_type>::value) {
-            py_value = as_unsigned<py_type>(src.ptr());
-        } else { // signed integer:
-            py_value = sizeof(T) <= sizeof(long)
-                ? (py_type) PyLong_AsLong(src.ptr())
-                : (py_type) PYBIND11_LONG_AS_LONGLONG(src.ptr());
+        } else {
+            handle src_or_index = src;
+#if PY_VERSION_HEX < 0x03080000
+            object index;
+            if (!PYBIND11_LONG_CHECK(src.ptr())) {  // So: index_check(src.ptr())
+                index = reinterpret_steal<object>(PyNumber_Index(src.ptr()));
+                if (!index) {
+                    PyErr_Clear();
+                    if (!convert)
+                        return false;
+                }
+                else {
+                    src_or_index = index;
+                }
+            }
+#endif
+            if (std::is_unsigned<py_type>::value) {
+                py_value = as_unsigned<py_type>(src_or_index.ptr());
+            } else { // signed integer:
+                py_value = sizeof(T) <= sizeof(long)
+                    ? (py_type) PyLong_AsLong(src_or_index.ptr())
+                    : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr());
+            }
         }
 
         // Python API reported an error
@@ -1056,15 +1073,8 @@ public:
         // Check to see if the conversion is valid (integers should match exactly)
         // Signed/unsigned checks happen elsewhere
         if (py_err || (std::is_integral<T>::value && sizeof(py_type) != sizeof(T) && py_value != (py_type) (T) py_value)) {
-            bool type_error = py_err && PyErr_ExceptionMatches(
-#if PY_VERSION_HEX < 0x03000000 && !defined(PYPY_VERSION)
-                PyExc_SystemError
-#else
-                PyExc_TypeError
-#endif
-            );
             PyErr_Clear();
-            if (type_error && convert && PyNumber_Check(src.ptr())) {
+            if (py_err && convert && PyNumber_Check(src.ptr())) {
                 auto tmp = reinterpret_steal<object>(std::is_floating_point<T>::value
                                                      ? PyNumber_Float(src.ptr())
                                                      : PyNumber_Long(src.ptr()));
diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py
index 0a46984..cb37dbc 100644
--- a/tests/test_builtin_casters.py
+++ b/tests/test_builtin_casters.py
@@ -252,22 +252,36 @@ def test_integer_casting():
 
 
 def test_int_convert():
-    class DeepThought(object):
+    class Int(object):
         def __int__(self):
             return 42
 
-    class ShallowThought(object):
+    class NotInt(object):
         pass
 
-    class FuzzyThought(object):
+    class Float(object):
         def __float__(self):
             return 41.99999
 
-    class IndexedThought(object):
+    class Index(object):
         def __index__(self):
             return 42
 
-    class RaisingThought(object):
+    class IntAndIndex(object):
+        def __int__(self):
+            return 42
+
+        def __index__(self):
+            return 0
+
+    class RaisingTypeErrorOnIndex(object):
+        def __index__(self):
+            raise TypeError
+
+        def __int__(self):
+            return 42
+
+    class RaisingValueErrorOnIndex(object):
         def __index__(self):
             raise ValueError
 
@@ -276,7 +290,7 @@ def test_int_convert():
 
     convert, noconvert = m.int_passthrough, m.int_passthrough_noconvert
 
-    def require_implicit(v):
+    def requires_conversion(v):
         pytest.raises(TypeError, noconvert, v)
 
     def cant_convert(v):
@@ -285,15 +299,21 @@ def test_int_convert():
     assert convert(7) == 7
     assert noconvert(7) == 7
     cant_convert(3.14159)
-    assert convert(DeepThought()) == 42
-    require_implicit(DeepThought())
-    cant_convert(ShallowThought())
-    cant_convert(FuzzyThought())
-    if env.PY >= (3, 8):
-        # Before Python 3.8, `int(obj)` does not pick up on `obj.__index__`
-        assert convert(IndexedThought()) == 42
-        assert noconvert(IndexedThought()) == 42
-        cant_convert(RaisingThought())  # no fall-back to `__int__`if `__index__` raises
+    assert convert(Int()) == 42
+    requires_conversion(Int())
+    cant_convert(NotInt())
+    cant_convert(Float())
+
+    # Before Python 3.8, `PyLong_AsLong` does not pick up on `obj.__index__`,
+    # but pybind11 "backports" this behavior.
+    assert convert(Index()) == 42
+    assert noconvert(Index()) == 42
+    assert convert(IntAndIndex()) == 0  # Fishy; `int(DoubleThought)` == 42
+    assert noconvert(IntAndIndex()) == 0
+    assert convert(RaisingTypeErrorOnIndex()) == 42
+    requires_conversion(RaisingTypeErrorOnIndex())
+    assert convert(RaisingValueErrorOnIndex()) == 42
+    requires_conversion(RaisingValueErrorOnIndex())
 
 
 def test_numpy_int_convert():