From 8f4eb006901553ccfa010b75660f979ab1ef31a4 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Thu, 15 Oct 2015 18:13:33 +0200 Subject: last breaking change: be consistent about the project name --- include/pybind11/numpy.h | 241 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 include/pybind11/numpy.h (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h new file mode 100644 index 0000000..8a11786 --- /dev/null +++ b/include/pybind11/numpy.h @@ -0,0 +1,241 @@ +/* + pybind11/numpy.h: Basic NumPy support, auto-vectorization support + + Copyright (c) 2015 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "pybind11.h" +#include "complex.h" + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4127) // warning C4127: Conditional expression is constant +#endif + +NAMESPACE_BEGIN(pybind11) + +template struct npy_format_descriptor { }; + +class array : public buffer { +public: + struct API { + enum Entries { + API_PyArray_Type = 2, + API_PyArray_DescrFromType = 45, + API_PyArray_FromAny = 69, + API_PyArray_NewCopy = 85, + API_PyArray_NewFromDescr = 94, + NPY_C_CONTIGUOUS = 0x0001, + NPY_F_CONTIGUOUS = 0x0002, + NPY_NPY_ARRAY_FORCECAST = 0x0010, + NPY_ENSURE_ARRAY = 0x0040, + NPY_BOOL=0, + NPY_BYTE, NPY_UBYTE, + NPY_SHORT, NPY_USHORT, + NPY_INT, NPY_UINT, + NPY_LONG, NPY_ULONG, + NPY_LONGLONG, NPY_ULONGLONG, + NPY_FLOAT, NPY_DOUBLE, NPY_LONGDOUBLE, + NPY_CFLOAT, NPY_CDOUBLE, NPY_CLONGDOUBLE + }; + + static API lookup() { + PyObject *numpy = PyImport_ImportModule("numpy.core.multiarray"); + PyObject *capsule = numpy ? PyObject_GetAttrString(numpy, "_ARRAY_API") : nullptr; +#if PY_MAJOR_VERSION >= 3 + void **api_ptr = (void **) (capsule ? PyCapsule_GetPointer(capsule, NULL) : nullptr); +#else + void **api_ptr = (void **) (capsule ? PyCObject_AsVoidPtr(capsule) : nullptr); +#endif + Py_XDECREF(capsule); + Py_XDECREF(numpy); + if (api_ptr == nullptr) + throw std::runtime_error("Could not acquire pointer to NumPy API!"); + API api; + api.PyArray_Type = (decltype(api.PyArray_Type)) api_ptr[API_PyArray_Type]; + api.PyArray_DescrFromType = (decltype(api.PyArray_DescrFromType)) api_ptr[API_PyArray_DescrFromType]; + api.PyArray_FromAny = (decltype(api.PyArray_FromAny)) api_ptr[API_PyArray_FromAny]; + api.PyArray_NewCopy = (decltype(api.PyArray_NewCopy)) api_ptr[API_PyArray_NewCopy]; + api.PyArray_NewFromDescr = (decltype(api.PyArray_NewFromDescr)) api_ptr[API_PyArray_NewFromDescr]; + return api; + } + + bool PyArray_Check(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type); } + + PyObject *(*PyArray_DescrFromType)(int); + PyObject *(*PyArray_NewFromDescr) + (PyTypeObject *, PyObject *, int, Py_intptr_t *, + Py_intptr_t *, void *, int, PyObject *); + PyObject *(*PyArray_NewCopy)(PyObject *, int); + PyTypeObject *PyArray_Type; + PyObject *(*PyArray_FromAny) (PyObject *, PyObject *, int, int, int, PyObject *); + }; + + PYBIND_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check) + + template array(size_t size, const Type *ptr) { + API& api = lookup_api(); + PyObject *descr = api.PyArray_DescrFromType(npy_format_descriptor::value); + if (descr == nullptr) + throw std::runtime_error("NumPy: unsupported buffer format!"); + Py_intptr_t shape = (Py_intptr_t) size; + PyObject *tmp = api.PyArray_NewFromDescr( + api.PyArray_Type, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr); + if (tmp == nullptr) + throw std::runtime_error("NumPy: unable to create array!"); + m_ptr = api.PyArray_NewCopy(tmp, -1 /* any order */); + Py_DECREF(tmp); + if (m_ptr == nullptr) + throw std::runtime_error("NumPy: unable to copy array!"); + } + + array(const buffer_info &info) { + API& api = lookup_api(); + if (info.format.size() != 1) + throw std::runtime_error("Unsupported buffer format!"); + int fmt = (int) info.format[0]; + if (info.format == "Zd") + fmt = API::NPY_CDOUBLE; + else if (info.format == "Zf") + fmt = API::NPY_CFLOAT; + PyObject *descr = api.PyArray_DescrFromType(fmt); + if (descr == nullptr) + throw std::runtime_error("NumPy: unsupported buffer format '" + info.format + "'!"); + PyObject *tmp = api.PyArray_NewFromDescr( + api.PyArray_Type, descr, info.ndim, (Py_intptr_t *) &info.shape[0], + (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr); + if (tmp == nullptr) + throw std::runtime_error("NumPy: unable to create array!"); + m_ptr = api.PyArray_NewCopy(tmp, -1 /* any order */); + Py_DECREF(tmp); + if (m_ptr == nullptr) + throw std::runtime_error("NumPy: unable to copy array!"); + } + +protected: + static API &lookup_api() { + static API api = API::lookup(); + return api; + } +}; + +template class array_t : public array { +public: + PYBIND_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); + array_t() : array() { } + static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } + PyObject *ensure(PyObject *ptr) { + if (ptr == nullptr) + return nullptr; + API &api = lookup_api(); + PyObject *descr = api.PyArray_DescrFromType(npy_format_descriptor::value); + return api.PyArray_FromAny(ptr, descr, 0, 0, + API::NPY_C_CONTIGUOUS | API::NPY_ENSURE_ARRAY | + API::NPY_NPY_ARRAY_FORCECAST, nullptr); + } +}; + +#define DECL_FMT(t, n) template<> struct npy_format_descriptor { enum { value = array::API::n }; } +DECL_FMT(int8_t, NPY_BYTE); DECL_FMT(uint8_t, NPY_UBYTE); DECL_FMT(int16_t, NPY_SHORT); +DECL_FMT(uint16_t, NPY_USHORT); DECL_FMT(int32_t, NPY_INT); DECL_FMT(uint32_t, NPY_UINT); +DECL_FMT(int64_t, NPY_LONGLONG); DECL_FMT(uint64_t, NPY_ULONGLONG); DECL_FMT(float, NPY_FLOAT); +DECL_FMT(double, NPY_DOUBLE); DECL_FMT(bool, NPY_BOOL); DECL_FMT(std::complex, NPY_CFLOAT); +DECL_FMT(std::complex, NPY_CDOUBLE); +#undef DECL_FMT + + +NAMESPACE_BEGIN(detail) +PYBIND_TYPE_CASTER_PYTYPE(array) +PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) +PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) +PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) +PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) +PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) +PYBIND_TYPE_CASTER_PYTYPE(array_t>) +PYBIND_TYPE_CASTER_PYTYPE(array_t>) +PYBIND_TYPE_CASTER_PYTYPE(array_t) + +template +struct vectorize_helper { + typename std::remove_reference::type f; + + template + vectorize_helper(T&&f) : f(std::forward(f)) { } + + object operator()(array_t... args) { + return run(args..., typename make_index_sequence::type()); + } + + template object run(array_t&... args, index_sequence) { + /* Request buffers from all parameters */ + const size_t N = sizeof...(Args); + std::array buffers {{ args.request()... }}; + + /* Determine dimensions parameters of output array */ + int ndim = 0; size_t count = 0; + std::vector shape; + for (size_t i=0; i count) { + ndim = buffers[i].ndim; + shape = buffers[i].shape; + count = buffers[i].count; + } + } + std::vector strides(ndim); + if (ndim > 0) { + strides[ndim-1] = sizeof(Return); + for (int i=ndim-1; i>0; --i) + strides[i-1] = strides[i] * shape[i]; + } + + /* Check if the parameters are actually compatible */ + for (size_t i=0; i result(count); + for (size_t i=0; i::value(), + ndim, shape, strides)); + } +}; + +NAMESPACE_END(detail) + +template +detail::vectorize_helper vectorize(const Func &f, Return (*) (Args ...)) { + return detail::vectorize_helper(f); +} + +template +detail::vectorize_helper vectorize(Return (*f) (Args ...)) { + return vectorize(f, f); +} + +template auto vectorize(func &&f) -> decltype( + vectorize(std::forward(f), (typename detail::remove_class::type::operator())>::type *) nullptr)) { + return vectorize(std::forward(f), (typename detail::remove_class::type::operator())>::type *) nullptr); +} + +NAMESPACE_END(pybind11) + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif -- cgit v1.2.3 From b1b714023a5ece9436c135a11c19837804d9f099 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 18 Oct 2015 16:48:30 +0200 Subject: consistent macro naming throughout the project --- include/pybind11/numpy.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8a11786..dfbdc49 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -76,7 +76,7 @@ public: PyObject *(*PyArray_FromAny) (PyObject *, PyObject *, int, int, int, PyObject *); }; - PYBIND_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check) + PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check) template array(size_t size, const Type *ptr) { API& api = lookup_api(); @@ -126,7 +126,7 @@ protected: template class array_t : public array { public: - PYBIND_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); + PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); array_t() : array() { } static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } PyObject *ensure(PyObject *ptr) { @@ -150,15 +150,15 @@ DECL_FMT(std::complex, NPY_CDOUBLE); NAMESPACE_BEGIN(detail) -PYBIND_TYPE_CASTER_PYTYPE(array) -PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) -PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) -PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) -PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) -PYBIND_TYPE_CASTER_PYTYPE(array_t) PYBIND_TYPE_CASTER_PYTYPE(array_t) -PYBIND_TYPE_CASTER_PYTYPE(array_t>) -PYBIND_TYPE_CASTER_PYTYPE(array_t>) -PYBIND_TYPE_CASTER_PYTYPE(array_t) +PYBIND11_TYPE_CASTER_PYTYPE(array) +PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) +PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) +PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) +PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) +PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) +PYBIND11_TYPE_CASTER_PYTYPE(array_t>) +PYBIND11_TYPE_CASTER_PYTYPE(array_t>) +PYBIND11_TYPE_CASTER_PYTYPE(array_t) template struct vectorize_helper { -- cgit v1.2.3 From ab92eb37650d3c59a4484b585b8dedde44dac89d Mon Sep 17 00:00:00 2001 From: Jan Dohl Date: Tue, 15 Dec 2015 04:16:49 +0100 Subject: Fixed py:array constructor from failing for complex types The array(const buffer_info &info) constructor fails when given complex types since their format string is 'Zd' or 'Zf' which has a length of two and causes an error here: if (info.format.size() != 1) throw std::runtime_error("Unsupported buffer format!"); Fixed by allowing format sizes of one and two. --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index dfbdc49..d3d8e10 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -96,7 +96,7 @@ public: array(const buffer_info &info) { API& api = lookup_api(); - if (info.format.size() != 1) + if ((info.format.size() < 1) || (info.format.size() > 2)) throw std::runtime_error("Unsupported buffer format!"); int fmt = (int) info.format[0]; if (info.format == "Zd") -- cgit v1.2.3 From d1a24823bc2e77643ac34f23b633a6c8edaf9333 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Wed, 16 Dec 2015 12:11:01 +0100 Subject: considerable simplifications to the Python type casters --- include/pybind11/numpy.h | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d3d8e10..d1196e0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -148,17 +148,7 @@ DECL_FMT(double, NPY_DOUBLE); DECL_FMT(bool, NPY_BOOL); DECL_FMT(std::complex, NPY_CDOUBLE); #undef DECL_FMT - NAMESPACE_BEGIN(detail) -PYBIND11_TYPE_CASTER_PYTYPE(array) -PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) -PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) -PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) -PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) -PYBIND11_TYPE_CASTER_PYTYPE(array_t) PYBIND11_TYPE_CASTER_PYTYPE(array_t) -PYBIND11_TYPE_CASTER_PYTYPE(array_t>) -PYBIND11_TYPE_CASTER_PYTYPE(array_t>) -PYBIND11_TYPE_CASTER_PYTYPE(array_t) template struct vectorize_helper { -- cgit v1.2.3 From 875df5528dd9e71bd3d3ea982ca16d1e30c420bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= Date: Mon, 28 Dec 2015 08:11:16 +0100 Subject: Make handle and related classes const correct. This gives handle classes a typical pointer semantics with respects to constness. --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d1196e0..add52de 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -129,7 +129,7 @@ public: PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); array_t() : array() { } static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } - PyObject *ensure(PyObject *ptr) { + static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) return nullptr; API &api = lookup_api(); -- cgit v1.2.3 From 4c1a6be4bd2a5eb8d72dcaa21bc45b1f9cc538f0 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 17 Jan 2016 22:36:37 +0100 Subject: minor cleanups in numpy.h, updated gitignore file for ninja --- include/pybind11/numpy.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index add52de..094f317 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -32,9 +32,9 @@ public: API_PyArray_NewFromDescr = 94, NPY_C_CONTIGUOUS = 0x0001, NPY_F_CONTIGUOUS = 0x0002, - NPY_NPY_ARRAY_FORCECAST = 0x0010, + NPY_ARRAY_FORCECAST = 0x0010, NPY_ENSURE_ARRAY = 0x0040, - NPY_BOOL=0, + NPY_BOOL = 0, NPY_BYTE, NPY_UBYTE, NPY_SHORT, NPY_USHORT, NPY_INT, NPY_UINT, @@ -136,7 +136,7 @@ public: PyObject *descr = api.PyArray_DescrFromType(npy_format_descriptor::value); return api.PyArray_FromAny(ptr, descr, 0, 0, API::NPY_C_CONTIGUOUS | API::NPY_ENSURE_ARRAY | - API::NPY_NPY_ARRAY_FORCECAST, nullptr); + API::NPY_ARRAY_FORCECAST, nullptr); } }; -- cgit v1.2.3 From 56e9f4942b38aa29e7677f21d959f619b1a1ca34 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 17 Jan 2016 22:36:38 +0100 Subject: improved signature names for subclasses of pybind11::handle --- include/pybind11/numpy.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 094f317..4b10532 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -150,6 +150,10 @@ DECL_FMT(std::complex, NPY_CDOUBLE); NAMESPACE_BEGIN(detail) +template struct handle_type_name> { + static PYBIND11_DESCR name() { return _("array[") + type_caster::name() + _("]"); } +}; + template struct vectorize_helper { typename std::remove_reference::type f; -- cgit v1.2.3 From 87dfad65443e4a64ab88850d3263cdd3905c0c2d Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 17 Jan 2016 22:36:38 +0100 Subject: avoid naming clashes with numpy (fixes #36) --- include/pybind11/numpy.h | 102 +++++++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 53 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 4b10532..27ebe7d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -30,65 +30,62 @@ public: API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, - NPY_C_CONTIGUOUS = 0x0001, - NPY_F_CONTIGUOUS = 0x0002, - NPY_ARRAY_FORCECAST = 0x0010, - NPY_ENSURE_ARRAY = 0x0040, - NPY_BOOL = 0, - NPY_BYTE, NPY_UBYTE, - NPY_SHORT, NPY_USHORT, - NPY_INT, NPY_UINT, - NPY_LONG, NPY_ULONG, - NPY_LONGLONG, NPY_ULONGLONG, - NPY_FLOAT, NPY_DOUBLE, NPY_LONGDOUBLE, - NPY_CFLOAT, NPY_CDOUBLE, NPY_CLONGDOUBLE + + NPY_C_CONTIGUOUS_ = 0x0001, + NPY_F_CONTIGUOUS_ = 0x0002, + NPY_ARRAY_FORCECAST_ = 0x0010, + NPY_ENSURE_ARRAY_ = 0x0040, + NPY_BOOL_ = 0, + NPY_BYTE_, NPY_UBYTE_, + NPY_SHORT_, NPY_USHORT_, + NPY_INT_, NPY_UINT_, + NPY_LONG_, NPY_ULONG_, + NPY_LONGLONG_, NPY_ULONGLONG_, + NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, + NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_ }; static API lookup() { - PyObject *numpy = PyImport_ImportModule("numpy.core.multiarray"); - PyObject *capsule = numpy ? PyObject_GetAttrString(numpy, "_ARRAY_API") : nullptr; + module m = module::import("numpy.core.multiarray"); + object c = (object) m.attr("_ARRAY_API"); #if PY_MAJOR_VERSION >= 3 - void **api_ptr = (void **) (capsule ? PyCapsule_GetPointer(capsule, NULL) : nullptr); + void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr); #else - void **api_ptr = (void **) (capsule ? PyCObject_AsVoidPtr(capsule) : nullptr); + void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); #endif - Py_XDECREF(capsule); - Py_XDECREF(numpy); - if (api_ptr == nullptr) - throw std::runtime_error("Could not acquire pointer to NumPy API!"); API api; - api.PyArray_Type = (decltype(api.PyArray_Type)) api_ptr[API_PyArray_Type]; - api.PyArray_DescrFromType = (decltype(api.PyArray_DescrFromType)) api_ptr[API_PyArray_DescrFromType]; - api.PyArray_FromAny = (decltype(api.PyArray_FromAny)) api_ptr[API_PyArray_FromAny]; - api.PyArray_NewCopy = (decltype(api.PyArray_NewCopy)) api_ptr[API_PyArray_NewCopy]; - api.PyArray_NewFromDescr = (decltype(api.PyArray_NewFromDescr)) api_ptr[API_PyArray_NewFromDescr]; + api.PyArray_Type_ = (decltype(api.PyArray_Type_)) api_ptr[API_PyArray_Type]; + api.PyArray_DescrFromType_ = (decltype(api.PyArray_DescrFromType_)) api_ptr[API_PyArray_DescrFromType]; + api.PyArray_FromAny_ = (decltype(api.PyArray_FromAny_)) api_ptr[API_PyArray_FromAny]; + api.PyArray_NewCopy_ = (decltype(api.PyArray_NewCopy_)) api_ptr[API_PyArray_NewCopy]; + api.PyArray_NewFromDescr_ = (decltype(api.PyArray_NewFromDescr_)) api_ptr[API_PyArray_NewFromDescr]; return api; } - bool PyArray_Check(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type); } + bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); } - PyObject *(*PyArray_DescrFromType)(int); - PyObject *(*PyArray_NewFromDescr) + PyObject *(*PyArray_DescrFromType_)(int); + PyObject *(*PyArray_NewFromDescr_) (PyTypeObject *, PyObject *, int, Py_intptr_t *, Py_intptr_t *, void *, int, PyObject *); - PyObject *(*PyArray_NewCopy)(PyObject *, int); - PyTypeObject *PyArray_Type; - PyObject *(*PyArray_FromAny) (PyObject *, PyObject *, int, int, int, PyObject *); + PyObject *(*PyArray_NewCopy_)(PyObject *, int); + PyTypeObject *PyArray_Type_; + PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); }; - PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check) + PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_) template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType(npy_format_descriptor::value); + PyObject *descr = api.PyArray_DescrFromType_(npy_format_descriptor::value); if (descr == nullptr) throw std::runtime_error("NumPy: unsupported buffer format!"); Py_intptr_t shape = (Py_intptr_t) size; - PyObject *tmp = api.PyArray_NewFromDescr( - api.PyArray_Type, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr); + PyObject *tmp = api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr); if (tmp == nullptr) throw std::runtime_error("NumPy: unable to create array!"); - m_ptr = api.PyArray_NewCopy(tmp, -1 /* any order */); + m_ptr = api.PyArray_NewCopy_(tmp, -1 /* any order */); Py_DECREF(tmp); if (m_ptr == nullptr) throw std::runtime_error("NumPy: unable to copy array!"); @@ -99,19 +96,18 @@ public: if ((info.format.size() < 1) || (info.format.size() > 2)) throw std::runtime_error("Unsupported buffer format!"); int fmt = (int) info.format[0]; - if (info.format == "Zd") - fmt = API::NPY_CDOUBLE; - else if (info.format == "Zf") - fmt = API::NPY_CFLOAT; - PyObject *descr = api.PyArray_DescrFromType(fmt); + if (info.format == "Zd") fmt = API::NPY_CDOUBLE_; + else if (info.format == "Zf") fmt = API::NPY_CFLOAT_; + + PyObject *descr = api.PyArray_DescrFromType_(fmt); if (descr == nullptr) throw std::runtime_error("NumPy: unsupported buffer format '" + info.format + "'!"); - PyObject *tmp = api.PyArray_NewFromDescr( - api.PyArray_Type, descr, info.ndim, (Py_intptr_t *) &info.shape[0], + PyObject *tmp = api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr, info.ndim, (Py_intptr_t *) &info.shape[0], (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr); if (tmp == nullptr) throw std::runtime_error("NumPy: unable to create array!"); - m_ptr = api.PyArray_NewCopy(tmp, -1 /* any order */); + m_ptr = api.PyArray_NewCopy_(tmp, -1 /* any order */); Py_DECREF(tmp); if (m_ptr == nullptr) throw std::runtime_error("NumPy: unable to copy array!"); @@ -133,19 +129,19 @@ public: if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType(npy_format_descriptor::value); - return api.PyArray_FromAny(ptr, descr, 0, 0, - API::NPY_C_CONTIGUOUS | API::NPY_ENSURE_ARRAY | - API::NPY_ARRAY_FORCECAST, nullptr); + PyObject *descr = api.PyArray_DescrFromType_(npy_format_descriptor::value); + return api.PyArray_FromAny_(ptr, descr, 0, 0, + API::NPY_C_CONTIGUOUS_ | API::NPY_ENSURE_ARRAY_ | + API::NPY_ARRAY_FORCECAST_, nullptr); } }; #define DECL_FMT(t, n) template<> struct npy_format_descriptor { enum { value = array::API::n }; } -DECL_FMT(int8_t, NPY_BYTE); DECL_FMT(uint8_t, NPY_UBYTE); DECL_FMT(int16_t, NPY_SHORT); -DECL_FMT(uint16_t, NPY_USHORT); DECL_FMT(int32_t, NPY_INT); DECL_FMT(uint32_t, NPY_UINT); -DECL_FMT(int64_t, NPY_LONGLONG); DECL_FMT(uint64_t, NPY_ULONGLONG); DECL_FMT(float, NPY_FLOAT); -DECL_FMT(double, NPY_DOUBLE); DECL_FMT(bool, NPY_BOOL); DECL_FMT(std::complex, NPY_CFLOAT); -DECL_FMT(std::complex, NPY_CDOUBLE); +DECL_FMT(int8_t, NPY_BYTE_); DECL_FMT(uint8_t, NPY_UBYTE_); DECL_FMT(int16_t, NPY_SHORT_); +DECL_FMT(uint16_t, NPY_USHORT_); DECL_FMT(int32_t, NPY_INT_); DECL_FMT(uint32_t, NPY_UINT_); +DECL_FMT(int64_t, NPY_LONGLONG_); DECL_FMT(uint64_t, NPY_ULONGLONG_); DECL_FMT(float, NPY_FLOAT_); +DECL_FMT(double, NPY_DOUBLE_); DECL_FMT(bool, NPY_BOOL_); DECL_FMT(std::complex, NPY_CFLOAT_); +DECL_FMT(std::complex, NPY_CDOUBLE_); #undef DECL_FMT NAMESPACE_BEGIN(detail) -- cgit v1.2.3 From 87187afe9156166863915cd9d46aab6d527dd689 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 17 Jan 2016 22:36:39 +0100 Subject: switch NumPy array to object API, avoid unnecessary copy operation in vectorize --- include/pybind11/numpy.h | 52 ++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 27ebe7d..3ff68f5 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -81,14 +81,13 @@ public: if (descr == nullptr) throw std::runtime_error("NumPy: unsupported buffer format!"); Py_intptr_t shape = (Py_intptr_t) size; - PyObject *tmp = api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr); - if (tmp == nullptr) + object tmp = object(api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); + if (ptr && tmp) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + if (!tmp) throw std::runtime_error("NumPy: unable to create array!"); - m_ptr = api.PyArray_NewCopy_(tmp, -1 /* any order */); - Py_DECREF(tmp); - if (m_ptr == nullptr) - throw std::runtime_error("NumPy: unable to copy array!"); + m_ptr = tmp.release(); } array(const buffer_info &info) { @@ -102,15 +101,14 @@ public: PyObject *descr = api.PyArray_DescrFromType_(fmt); if (descr == nullptr) throw std::runtime_error("NumPy: unsupported buffer format '" + info.format + "'!"); - PyObject *tmp = api.PyArray_NewFromDescr_( + object tmp(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, info.ndim, (Py_intptr_t *) &info.shape[0], - (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr); - if (tmp == nullptr) + (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); + if (info.ptr && tmp) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + if (!tmp) throw std::runtime_error("NumPy: unable to create array!"); - m_ptr = api.PyArray_NewCopy_(tmp, -1 /* any order */); - Py_DECREF(tmp); - if (m_ptr == nullptr) - throw std::runtime_error("NumPy: unable to copy array!"); + m_ptr = tmp.release(); } protected: @@ -184,25 +182,27 @@ struct vectorize_helper { } /* Check if the parameters are actually compatible */ - for (size_t i=0; i result(count); - for (size_t i=0; i::value(), ndim, shape, strides)); + + buffer_info buf = result.request(); + Return *output = (Return *) buf.ptr; + + /* Call the function */ + for (size_t i=0; i Date: Sun, 17 Jan 2016 22:36:40 +0100 Subject: moved lifetime management of Py_buffer to pybind11::buffer_info, renamed count->size to match NumPy naming (fixes #34) --- include/pybind11/numpy.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3ff68f5..11bb47f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -165,13 +165,13 @@ struct vectorize_helper { std::array buffers {{ args.request()... }}; /* Determine dimensions parameters of output array */ - int ndim = 0; size_t count = 0; + int ndim = 0; size_t size = 0; std::vector shape; for (size_t i=0; i count) { + if (buffers[i].size > size) { ndim = buffers[i].ndim; shape = buffers[i].shape; - count = buffers[i].count; + size = buffers[i].size; } } std::vector strides(ndim); @@ -183,10 +183,10 @@ struct vectorize_helper { /* Check if the parameters are actually compatible */ for (size_t i=0; i Date: Sun, 17 Jan 2016 22:36:41 +0100 Subject: numpy.h: fixed a leak, added some comments to buffer_info --- include/pybind11/numpy.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 11bb47f..2365a60 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -128,9 +128,11 @@ public: return nullptr; API &api = lookup_api(); PyObject *descr = api.PyArray_DescrFromType_(npy_format_descriptor::value); - return api.PyArray_FromAny_(ptr, descr, 0, 0, - API::NPY_C_CONTIGUOUS_ | API::NPY_ENSURE_ARRAY_ | - API::NPY_ARRAY_FORCECAST_, nullptr); + PyObject *result = api.PyArray_FromAny_( + ptr, descr, 0, 0, API::NPY_C_CONTIGUOUS_ | API::NPY_ENSURE_ARRAY_ + | API::NPY_ARRAY_FORCECAST_, nullptr); + Py_DECREF(ptr); + return result; } }; -- cgit v1.2.3 From 48548ea4a5735acfb86c67354d3ea6a728169740 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 17 Jan 2016 22:36:44 +0100 Subject: general cleanup of the codebase - new pybind11::base<> attribute to indicate a subclass relationship - unified infrastructure for parsing variadic arguments in class_ and cpp_function - use 'handle' and 'object' more consistently everywhere --- include/pybind11/numpy.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2365a60..0aaac96 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -79,36 +79,36 @@ public: API& api = lookup_api(); PyObject *descr = api.PyArray_DescrFromType_(npy_format_descriptor::value); if (descr == nullptr) - throw std::runtime_error("NumPy: unsupported buffer format!"); + pybind11_fail("NumPy: unsupported buffer format!"); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); if (ptr && tmp) tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); if (!tmp) - throw std::runtime_error("NumPy: unable to create array!"); - m_ptr = tmp.release(); + pybind11_fail("NumPy: unable to create array!"); + m_ptr = tmp.release().ptr(); } array(const buffer_info &info) { API& api = lookup_api(); if ((info.format.size() < 1) || (info.format.size() > 2)) - throw std::runtime_error("Unsupported buffer format!"); + pybind11_fail("Unsupported buffer format!"); int fmt = (int) info.format[0]; if (info.format == "Zd") fmt = API::NPY_CDOUBLE_; else if (info.format == "Zf") fmt = API::NPY_CFLOAT_; PyObject *descr = api.PyArray_DescrFromType_(fmt); if (descr == nullptr) - throw std::runtime_error("NumPy: unsupported buffer format '" + info.format + "'!"); + pybind11_fail("NumPy: unsupported buffer format '" + info.format + "'!"); object tmp(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, info.ndim, (Py_intptr_t *) &info.shape[0], (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); if (info.ptr && tmp) tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); if (!tmp) - throw std::runtime_error("NumPy: unable to create array!"); - m_ptr = tmp.release(); + pybind11_fail("NumPy: unable to create array!"); + m_ptr = tmp.release().ptr(); } protected: @@ -186,7 +186,7 @@ struct vectorize_helper { /* Check if the parameters are actually compatible */ for (size_t i=0; i Date: Thu, 11 Feb 2016 10:47:11 +0100 Subject: NumPy-style broadcasting support in pybind11::vectorize --- include/pybind11/numpy.h | 204 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 184 insertions(+), 20 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0aaac96..44a0cb1 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -11,6 +11,8 @@ #include "pybind11.h" #include "complex.h" +#include +#include #if defined(_MSC_VER) #pragma warning(push) @@ -146,10 +148,158 @@ DECL_FMT(std::complex, NPY_CDOUBLE_); NAMESPACE_BEGIN(detail) +template +using array_iterator = typename std::add_pointer::type; + +template +array_iterator array_begin(const buffer_info& buffer) { + return array_iterator(reinterpret_cast(buffer.ptr)); +} + +template +array_iterator array_end(const buffer_info& buffer) { + return array_iterator(reinterpret_cast(buffer.ptr) + buffer.size); +} + +class common_iterator { + +public: + + using container_type = std::vector; + using value_type = container_type::value_type; + using size_type = container_type::size_type; + + common_iterator() : p_ptr(0), m_strides() {} + common_iterator(void* ptr, const container_type& strides, const std::vector& shape) + : p_ptr(reinterpret_cast(ptr)), m_strides(strides.size()) { + m_strides.back() = static_cast(strides.back()); + for (size_type i = m_strides.size() - 1; i != 0; --i) { + size_type j = i - 1; + value_type s = static_cast(shape[i]); + m_strides[j] = strides[j] + m_strides[i] - strides[i] * s; + } + } + + void increment(size_type dim) { + p_ptr += m_strides[dim]; + } + + void* data() const { + return p_ptr; + } + +private: + + char* p_ptr; + container_type m_strides; +}; + +template +class multi_array_iterator { + +public: + + using container_type = std::vector; + + multi_array_iterator(const std::array& buffers, + const std::vector& shape) + : m_shape(shape.size()), m_index(shape.size(), 0), m_common_iterator() { + // Manual copy to avoid conversion warning if using std::copy + for (size_t i = 0; i < shape.size(); ++i) { + m_shape[i] = static_cast(shape[i]); + } + + container_type strides(shape.size()); + for (size_t i = 0; i < N; ++i) { + init_common_iterator(buffers[i], shape, m_common_iterator[i], strides); + } + } + + multi_array_iterator& operator++() { + for (size_t j = m_index.size(); j != 0; --j) { + size_t i = j - 1; + if (++m_index[i] != m_shape[i]) { + increment_common_iterator(i); + break; + } + else { + m_index[i] = 0; + } + } + return *this; + } + + template + const T& data() const { + return *reinterpret_cast(m_common_iterator[K].data()); + } + +private: + + using common_iter = common_iterator; + + void init_common_iterator(const buffer_info& buffer, const std::vector& shape, common_iter& iterator, container_type& strides) { + auto buffer_shape_iter = buffer.shape.rbegin(); + auto buffer_strides_iter = buffer.strides.rbegin(); + auto shape_iter = shape.rbegin(); + auto strides_iter = strides.rbegin(); + + while (buffer_shape_iter != buffer.shape.rend()) { + if (*shape_iter == *buffer_shape_iter) + *strides_iter = static_cast(*buffer_strides_iter); + else + *strides_iter = 0; + + ++buffer_shape_iter; + ++buffer_strides_iter; + ++shape_iter; + ++strides_iter; + } + + std::fill(strides_iter, strides.rend(), 0); + iterator = common_iter(buffer.ptr, strides, shape); + } + + void increment_common_iterator(size_t dim) { + std::for_each(m_common_iterator.begin(), m_common_iterator.end(), [=](common_iter& iter) { + iter.increment(dim); + }); + } + + container_type m_shape; + container_type m_index; + std::array m_common_iterator; +}; + template struct handle_type_name> { static PYBIND11_DESCR name() { return _("array[") + type_caster::name() + _("]"); } }; +template +bool broadcast(const std::array& buffers, int& ndim, std::vector& shape) { + ndim = std::accumulate(buffers.begin(), buffers.end(), 0, [](int res, const buffer_info& buf) { + return std::max(res, buf.ndim); + }); + + shape = std::vector(static_cast(ndim), 1); + bool trivial_broadcast = true; + for (size_t i = 0; i < N; ++i) { + auto res_iter = shape.rbegin(); + bool i_trivial_broadcast = (buffers[i].size == 1) || (buffers[i].ndim == ndim); + for (auto shape_iter = buffers[i].shape.rbegin(); shape_iter != buffers[i].shape.rend(); ++shape_iter, ++res_iter) { + if (*res_iter == 1) { + *res_iter = *shape_iter; + } + else if ((*shape_iter != 1) && (*res_iter != *shape_iter)) { + pybind11_fail("pybind11::vectorize: incompatible size/dimension of inputs!"); + } + i_trivial_broadcast = i_trivial_broadcast && (*res_iter == *shape_iter); + } + trivial_broadcast = trivial_broadcast && i_trivial_broadcast; + } + return trivial_broadcast; +} + template struct vectorize_helper { typename std::remove_reference::type f; @@ -161,33 +311,28 @@ struct vectorize_helper { return run(args..., typename make_index_sequence::type()); } - template object run(array_t&... args, index_sequence) { + template object run(array_t&... args, index_sequence index) { /* Request buffers from all parameters */ const size_t N = sizeof...(Args); + std::array buffers {{ args.request()... }}; /* Determine dimensions parameters of output array */ - int ndim = 0; size_t size = 0; - std::vector shape; - for (size_t i=0; i size) { - ndim = buffers[i].ndim; - shape = buffers[i].shape; - size = buffers[i].size; - } - } + int ndim = 0; + std::vector shape(0); + bool trivial_broadcast = broadcast(buffers, ndim, shape); + + size_t size = 1; std::vector strides(ndim); if (ndim > 0) { strides[ndim-1] = sizeof(Return); - for (int i=ndim-1; i>0; --i) - strides[i-1] = strides[i] * shape[i]; + for (int i = ndim - 1; i > 0; --i) { + strides[i - 1] = strides[i] * shape[i]; + size *= shape[i]; + } + size *= shape[0]; } - /* Check if the parameters are actually compatible */ - for (size_t i=0; i(buffers, buf, index); + } return result; } + + template + void apply_broadcast(const std::array& buffers, buffer_info& output, index_sequence) { + using input_iterator = multi_array_iterator; + using output_iterator = array_iterator; + + input_iterator input_iter(buffers, output.shape); + output_iterator output_end = array_end(output); + + for (output_iterator iter = array_begin(output); iter != output_end; ++iter, ++input_iter) { + *iter = f((input_iter.template data())...); + } + } }; NAMESPACE_END(detail) -- cgit v1.2.3 From d6e4cef65f118b1a3167b89452161dfc83c81166 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sat, 20 Feb 2016 12:17:17 +0100 Subject: minor formatting changes, removed missing header files referenced in setup.py --- include/pybind11/numpy.h | 64 +++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 34 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 44a0cb1..3386876 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -162,14 +162,13 @@ array_iterator array_end(const buffer_info& buffer) { } class common_iterator { - public: - using container_type = std::vector; using value_type = container_type::value_type; using size_type = container_type::size_type; common_iterator() : p_ptr(0), m_strides() {} + common_iterator(void* ptr, const container_type& strides, const std::vector& shape) : p_ptr(reinterpret_cast(ptr)), m_strides(strides.size()) { m_strides.back() = static_cast(strides.back()); @@ -189,30 +188,26 @@ public: } private: - char* p_ptr; container_type m_strides; }; -template -class multi_array_iterator { - +template class multi_array_iterator { public: - using container_type = std::vector; - multi_array_iterator(const std::array& buffers, - const std::vector& shape) - : m_shape(shape.size()), m_index(shape.size(), 0), m_common_iterator() { + multi_array_iterator(const std::array &buffers, + const std::vector &shape) + : m_shape(shape.size()), m_index(shape.size(), 0), + m_common_iterator() { + // Manual copy to avoid conversion warning if using std::copy - for (size_t i = 0; i < shape.size(); ++i) { + for (size_t i = 0; i < shape.size(); ++i) m_shape[i] = static_cast(shape[i]); - } container_type strides(shape.size()); - for (size_t i = 0; i < N; ++i) { + for (size_t i = 0; i < N; ++i) init_common_iterator(buffers[i], shape, m_common_iterator[i], strides); - } } multi_array_iterator& operator++() { @@ -221,16 +216,14 @@ public: if (++m_index[i] != m_shape[i]) { increment_common_iterator(i); break; - } - else { + } else { m_index[i] = 0; } } return *this; } - template - const T& data() const { + template const T& data() const { return *reinterpret_cast(m_common_iterator[K].data()); } @@ -238,7 +231,9 @@ private: using common_iter = common_iterator; - void init_common_iterator(const buffer_info& buffer, const std::vector& shape, common_iter& iterator, container_type& strides) { + void init_common_iterator(const buffer_info &buffer, + const std::vector &shape, + common_iter &iterator, container_type &strides) { auto buffer_shape_iter = buffer.shape.rbegin(); auto buffer_strides_iter = buffer.strides.rbegin(); auto shape_iter = shape.rbegin(); @@ -261,9 +256,8 @@ private: } void increment_common_iterator(size_t dim) { - std::for_each(m_common_iterator.begin(), m_common_iterator.end(), [=](common_iter& iter) { + for (auto &iter : m_common_iterator) iter.increment(dim); - }); } container_type m_shape; @@ -271,10 +265,6 @@ private: std::array m_common_iterator; }; -template struct handle_type_name> { - static PYBIND11_DESCR name() { return _("array[") + type_caster::name() + _("]"); } -}; - template bool broadcast(const std::array& buffers, int& ndim, std::vector& shape) { ndim = std::accumulate(buffers.begin(), buffers.end(), 0, [](int res, const buffer_info& buf) { @@ -286,13 +276,14 @@ bool broadcast(const std::array& buffers, int& ndim, std::vector for (size_t i = 0; i < N; ++i) { auto res_iter = shape.rbegin(); bool i_trivial_broadcast = (buffers[i].size == 1) || (buffers[i].ndim == ndim); - for (auto shape_iter = buffers[i].shape.rbegin(); shape_iter != buffers[i].shape.rend(); ++shape_iter, ++res_iter) { - if (*res_iter == 1) { + for (auto shape_iter = buffers[i].shape.rbegin(); + shape_iter != buffers[i].shape.rend(); ++shape_iter, ++res_iter) { + + if (*res_iter == 1) *res_iter = *shape_iter; - } - else if ((*shape_iter != 1) && (*res_iter != *shape_iter)) { + else if ((*shape_iter != 1) && (*res_iter != *shape_iter)) pybind11_fail("pybind11::vectorize: incompatible size/dimension of inputs!"); - } + i_trivial_broadcast = i_trivial_broadcast && (*res_iter == *shape_iter); } trivial_broadcast = trivial_broadcast && i_trivial_broadcast; @@ -350,8 +341,7 @@ struct vectorize_helper { ? *((Args *) buffers[Index].ptr) : ((Args *) buffers[Index].ptr)[i])...); } - } - else { + } else { apply_broadcast(buffers, buf, index); } @@ -359,19 +349,25 @@ struct vectorize_helper { } template - void apply_broadcast(const std::array& buffers, buffer_info& output, index_sequence) { + void apply_broadcast(const std::array &buffers, + buffer_info &output, index_sequence) { using input_iterator = multi_array_iterator; using output_iterator = array_iterator; input_iterator input_iter(buffers, output.shape); output_iterator output_end = array_end(output); - for (output_iterator iter = array_begin(output); iter != output_end; ++iter, ++input_iter) { + for (output_iterator iter = array_begin(output); + iter != output_end; ++iter, ++input_iter) { *iter = f((input_iter.template data())...); } } }; +template struct handle_type_name> { + static PYBIND11_DESCR name() { return _("array[") + type_caster::name() + _("]"); } +}; + NAMESPACE_END(detail) template -- cgit v1.2.3 From 8cb6cb33ef10257a1f45e038897c91a558745943 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 17 Apr 2016 20:21:41 +0200 Subject: minor cleanups in common.h; updated author info and copyright year --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3386876..38d30bf 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1,7 +1,7 @@ /* pybind11/numpy.h: Basic NumPy support, auto-vectorization support - Copyright (c) 2015 Wenzel Jakob + Copyright (c) 2016 Wenzel Jakob All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. -- cgit v1.2.3 From 876eeab4ca41c68020a4279a89dfb70e36b58cc7 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Wed, 4 May 2016 22:22:48 +0200 Subject: redesigned format_descriptor<> and npy_format_descriptor<> This somewhat heavyweight solution will avoid size_t/long long/long/int mismatches on various platforms once and for all. The previous template overloads could e.g. not handle size_t on Darwin. One gotcha: the 'format_descriptor::value()' syntax changed to just 'format_descriptor::value' --- include/pybind11/numpy.h | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 38d30bf..cf46c33 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -21,7 +21,7 @@ NAMESPACE_BEGIN(pybind11) -template struct npy_format_descriptor { }; +template struct npy_format_descriptor { }; class array : public buffer { public: @@ -138,12 +138,20 @@ public: } }; +template struct npy_format_descriptor::value>::type> { +private: + constexpr static const int values[] = { + array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_, array::API::NPY_USHORT_, + array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; +public: + enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; +}; +template constexpr const int npy_format_descriptor< + T, typename std::enable_if::value>::type>::values[8]; + #define DECL_FMT(t, n) template<> struct npy_format_descriptor { enum { value = array::API::n }; } -DECL_FMT(int8_t, NPY_BYTE_); DECL_FMT(uint8_t, NPY_UBYTE_); DECL_FMT(int16_t, NPY_SHORT_); -DECL_FMT(uint16_t, NPY_USHORT_); DECL_FMT(int32_t, NPY_INT_); DECL_FMT(uint32_t, NPY_UINT_); -DECL_FMT(int64_t, NPY_LONGLONG_); DECL_FMT(uint64_t, NPY_ULONGLONG_); DECL_FMT(float, NPY_FLOAT_); -DECL_FMT(double, NPY_DOUBLE_); DECL_FMT(bool, NPY_BOOL_); DECL_FMT(std::complex, NPY_CFLOAT_); -DECL_FMT(std::complex, NPY_CDOUBLE_); +DECL_FMT(float, NPY_FLOAT_); DECL_FMT(double, NPY_DOUBLE_); DECL_FMT(bool, NPY_BOOL_); +DECL_FMT(std::complex, NPY_CFLOAT_); DECL_FMT(std::complex, NPY_CDOUBLE_); #undef DECL_FMT NAMESPACE_BEGIN(detail) @@ -328,7 +336,7 @@ struct vectorize_helper { return cast(f(*((Args *) buffers[Index].ptr)...)); array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::value(), + format_descriptor::value, ndim, shape, strides)); buffer_info buf = result.request(); -- cgit v1.2.3 From f1032df8916cfd866b8d1843169d9142e50c99f8 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Thu, 5 May 2016 10:00:00 +0200 Subject: only do numpy contiguous C/Fortran array conversion when explicitly requested --- include/pybind11/numpy.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index cf46c33..42d027a 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -77,6 +77,11 @@ public: PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_) + enum { + c_style = API::NPY_C_CONTIGUOUS_, + f_style = API::NPY_F_CONTIGUOUS_ + }; + template array(size_t size, const Type *ptr) { API& api = lookup_api(); PyObject *descr = api.PyArray_DescrFromType_(npy_format_descriptor::value); @@ -120,7 +125,7 @@ protected: } }; -template class array_t : public array { +template class array_t : public array { public: PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); array_t() : array() { } @@ -131,8 +136,9 @@ public: API &api = lookup_api(); PyObject *descr = api.PyArray_DescrFromType_(npy_format_descriptor::value); PyObject *result = api.PyArray_FromAny_( - ptr, descr, 0, 0, API::NPY_C_CONTIGUOUS_ | API::NPY_ENSURE_ARRAY_ - | API::NPY_ARRAY_FORCECAST_, nullptr); + ptr, descr, 0, 0, + API::NPY_ENSURE_ARRAY_ | API::NPY_ARRAY_FORCECAST_ | ExtraFlags, + nullptr); Py_DECREF(ptr); return result; } -- cgit v1.2.3 From 9e0a0568fed8c05ba7bb8d5101af7b6c407fb4eb Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Thu, 5 May 2016 20:33:54 +0200 Subject: transparent conversion of dense and sparse Eigen types --- include/pybind11/numpy.h | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 42d027a..b35790b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1,5 +1,5 @@ /* - pybind11/numpy.h: Basic NumPy support, auto-vectorization support + pybind11/numpy.h: Basic NumPy support, vectorize() wrapper Copyright (c) 2016 Wenzel Jakob @@ -20,8 +20,7 @@ #endif NAMESPACE_BEGIN(pybind11) - -template struct npy_format_descriptor { }; +namespace detail { template struct npy_format_descriptor { }; } class array : public buffer { public: @@ -84,7 +83,7 @@ public: template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(npy_format_descriptor::value); + PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); if (descr == nullptr) pybind11_fail("NumPy: unsupported buffer format!"); Py_intptr_t shape = (Py_intptr_t) size; @@ -134,7 +133,7 @@ public: if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(npy_format_descriptor::value); + PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); PyObject *result = api.PyArray_FromAny_( ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | API::NPY_ARRAY_FORCECAST_ | ExtraFlags, @@ -144,6 +143,8 @@ public: } }; +NAMESPACE_BEGIN(detail) + template struct npy_format_descriptor::value>::type> { private: constexpr static const int values[] = { @@ -151,17 +152,21 @@ private: array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; public: enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; + template ::value, int>::type = 0> + static PYBIND11_DESCR name() { return _("int") + _(); } + template ::value, int>::type = 0> + static PYBIND11_DESCR name() { return _("uint") + _(); } }; template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; -#define DECL_FMT(t, n) template<> struct npy_format_descriptor { enum { value = array::API::n }; } -DECL_FMT(float, NPY_FLOAT_); DECL_FMT(double, NPY_DOUBLE_); DECL_FMT(bool, NPY_BOOL_); -DECL_FMT(std::complex, NPY_CFLOAT_); DECL_FMT(std::complex, NPY_CDOUBLE_); +#define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ + enum { value = array::API::NumPyName }; \ + static PYBIND11_DESCR name() { return _(Name); } } +DECL_FMT(float, NPY_FLOAT_, "float32"); DECL_FMT(double, NPY_DOUBLE_, "float64"); DECL_FMT(bool, NPY_BOOL_, "bool"); +DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #undef DECL_FMT -NAMESPACE_BEGIN(detail) - template using array_iterator = typename std::add_pointer::type; @@ -348,7 +353,7 @@ struct vectorize_helper { buffer_info buf = result.request(); Return *output = (Return *) buf.ptr; - if(trivial_broadcast) { + if (trivial_broadcast) { /* Call the function */ for (size_t i=0; i struct handle_type_name> { - static PYBIND11_DESCR name() { return _("array[") + type_caster::name() + _("]"); } + static PYBIND11_DESCR name() { return _("numpy.ndarray[dtype=") + type_caster::name() + _("]"); } }; NAMESPACE_END(detail) -- cgit v1.2.3 From a580ed8c47d238d7481a8cea79f8d598818e3609 Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 11 May 2016 14:45:01 +0200 Subject: Compilation issue fixed --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b35790b..eb4ee60 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -147,7 +147,7 @@ NAMESPACE_BEGIN(detail) template struct npy_format_descriptor::value>::type> { private: - constexpr static const int values[] = { + constexpr static const int values[8] = { array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_, array::API::NPY_USHORT_, array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; public: -- cgit v1.2.3 From a63d93ba443b9aaaabb5ecf985144197fc80aafb Mon Sep 17 00:00:00 2001 From: Johan Mabille Date: Wed, 11 May 2016 11:25:15 +0200 Subject: constructor fix --- include/pybind11/numpy.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index eb4ee60..0b0e0ee 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -128,6 +128,7 @@ template class array_t : public array { public: PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); array_t() : array() { } + array_t(const buffer_info& info) : array(info) {} static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) -- cgit v1.2.3 From b47a9de03542df3743f252b935c52e0d8582a77c Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Thu, 19 May 2016 16:02:09 +0200 Subject: ability to prevent force casts in numpy arguments --- include/pybind11/numpy.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0b0e0ee..f97c790 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -78,7 +78,8 @@ public: enum { c_style = API::NPY_C_CONTIGUOUS_, - f_style = API::NPY_F_CONTIGUOUS_ + f_style = API::NPY_F_CONTIGUOUS_, + forcecast = API::NPY_ARRAY_FORCECAST_ }; template array(size_t size, const Type *ptr) { @@ -124,7 +125,7 @@ protected: } }; -template class array_t : public array { +template class array_t : public array { public: PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); array_t() : array() { } @@ -135,10 +136,9 @@ public: return nullptr; API &api = lookup_api(); PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); - PyObject *result = api.PyArray_FromAny_( - ptr, descr, 0, 0, - API::NPY_ENSURE_ARRAY_ | API::NPY_ARRAY_FORCECAST_ | ExtraFlags, - nullptr); + PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + if (!result) + PyErr_Clear(); Py_DECREF(ptr); return result; } @@ -318,11 +318,11 @@ struct vectorize_helper { template vectorize_helper(T&&f) : f(std::forward(f)) { } - object operator()(array_t... args) { + object operator()(array_t... args) { return run(args..., typename make_index_sequence::type()); } - template object run(array_t&... args, index_sequence index) { + template object run(array_t&... args, index_sequence index) { /* Request buffers from all parameters */ const size_t N = sizeof...(Args); @@ -332,7 +332,7 @@ struct vectorize_helper { int ndim = 0; std::vector shape(0); bool trivial_broadcast = broadcast(buffers, ndim, shape); - + size_t size = 1; std::vector strides(ndim); if (ndim > 0) { @@ -384,7 +384,7 @@ struct vectorize_helper { } }; -template struct handle_type_name> { +template struct handle_type_name> { static PYBIND11_DESCR name() { return _("numpy.ndarray[dtype=") + type_caster::name() + _("]"); } }; -- cgit v1.2.3 From 0a07805ab663b981d72bbe2663b946f8c18e67cf Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 29 May 2016 13:40:40 +0200 Subject: fixed many conversion warnings on clang --- include/pybind11/numpy.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index f97c790..0180d95 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -109,7 +109,7 @@ public: if (descr == nullptr) pybind11_fail("NumPy: unsupported buffer format '" + info.format + "'!"); object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, info.ndim, (Py_intptr_t *) &info.shape[0], + api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0], (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); if (info.ptr && tmp) tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); @@ -261,7 +261,7 @@ private: while (buffer_shape_iter != buffer.shape.rend()) { if (*shape_iter == *buffer_shape_iter) - *strides_iter = static_cast(*buffer_strides_iter); + *strides_iter = static_cast(*buffer_strides_iter); else *strides_iter = 0; @@ -286,12 +286,12 @@ private: }; template -bool broadcast(const std::array& buffers, int& ndim, std::vector& shape) { - ndim = std::accumulate(buffers.begin(), buffers.end(), 0, [](int res, const buffer_info& buf) { +bool broadcast(const std::array& buffers, size_t& ndim, std::vector& shape) { + ndim = std::accumulate(buffers.begin(), buffers.end(), size_t(0), [](size_t res, const buffer_info& buf) { return std::max(res, buf.ndim); }); - shape = std::vector(static_cast(ndim), 1); + shape = std::vector(ndim, 1); bool trivial_broadcast = true; for (size_t i = 0; i < N; ++i) { auto res_iter = shape.rbegin(); @@ -329,7 +329,7 @@ struct vectorize_helper { std::array buffers {{ args.request()... }}; /* Determine dimensions parameters of output array */ - int ndim = 0; + size_t ndim = 0; std::vector shape(0); bool trivial_broadcast = broadcast(buffers, ndim, shape); @@ -337,7 +337,7 @@ struct vectorize_helper { std::vector strides(ndim); if (ndim > 0) { strides[ndim-1] = sizeof(Return); - for (int i = ndim - 1; i > 0; --i) { + for (size_t i = ndim - 1; i > 0; --i) { strides[i - 1] = strides[i] * shape[i]; size *= shape[i]; } -- cgit v1.2.3 From ed23dda93b3735e220c5931f646dc8193d2d1b31 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Thu, 4 Aug 2016 01:40:40 +0200 Subject: Adopt PEP 484 type hints for C++ types exported to Python --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0180d95..5d355e2 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -385,7 +385,7 @@ struct vectorize_helper { }; template struct handle_type_name> { - static PYBIND11_DESCR name() { return _("numpy.ndarray[dtype=") + type_caster::name() + _("]"); } + static PYBIND11_DESCR name() { return _("numpy.ndarray[") + type_caster::name() + _("]"); } }; NAMESPACE_END(detail) -- cgit v1.2.3 From 3dd325b77280be9ef83cfd986309d2489a709686 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 18 Jun 2016 00:46:12 +0100 Subject: Change npy_format_descriptor typenum to static fn --- include/pybind11/numpy.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5d355e2..e8182ac 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -84,7 +84,7 @@ public: template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); + PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::typenum()); if (descr == nullptr) pybind11_fail("NumPy: unsupported buffer format!"); Py_intptr_t shape = (Py_intptr_t) size; @@ -135,7 +135,7 @@ public: if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); + PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::typenum()); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -152,7 +152,7 @@ private: array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_, array::API::NPY_USHORT_, array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; public: - enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; + static int typenum() { return values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)]; } template ::value, int>::type = 0> static PYBIND11_DESCR name() { return _("int") + _(); } template ::value, int>::type = 0> @@ -162,10 +162,13 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - enum { value = array::API::NumPyName }; \ + static int typenum() { return array::API::NumPyName; } \ static PYBIND11_DESCR name() { return _(Name); } } -DECL_FMT(float, NPY_FLOAT_, "float32"); DECL_FMT(double, NPY_DOUBLE_, "float64"); DECL_FMT(bool, NPY_BOOL_, "bool"); -DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); +DECL_FMT(float, NPY_FLOAT_, "float32"); +DECL_FMT(double, NPY_DOUBLE_, "float64"); +DECL_FMT(bool, NPY_BOOL_, "bool"); +DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); +DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #undef DECL_FMT template -- cgit v1.2.3 From 42ad3284815b06cc74fa9d19d767a73d86e73546 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:39:41 +0100 Subject: Change format_descriptor::value to a static func --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e8182ac..7dff28d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -351,7 +351,7 @@ struct vectorize_helper { return cast(f(*((Args *) buffers[Index].ptr)...)); array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::value, + format_descriptor::value(), ndim, shape, strides)); buffer_info buf = result.request(); -- cgit v1.2.3 From ea2755ccdc25c6f5ec8c4061d2e69d2e56521442 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:44:20 +0100 Subject: Use a macro for numpy API definitions --- include/pybind11/numpy.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 7dff28d..8e8abe3 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -55,11 +55,13 @@ public: void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); #endif API api; - api.PyArray_Type_ = (decltype(api.PyArray_Type_)) api_ptr[API_PyArray_Type]; - api.PyArray_DescrFromType_ = (decltype(api.PyArray_DescrFromType_)) api_ptr[API_PyArray_DescrFromType]; - api.PyArray_FromAny_ = (decltype(api.PyArray_FromAny_)) api_ptr[API_PyArray_FromAny]; - api.PyArray_NewCopy_ = (decltype(api.PyArray_NewCopy_)) api_ptr[API_PyArray_NewCopy]; - api.PyArray_NewFromDescr_ = (decltype(api.PyArray_NewFromDescr_)) api_ptr[API_PyArray_NewFromDescr]; +#define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; + DECL_NPY_API(PyArray_Type); + DECL_NPY_API(PyArray_DescrFromType); + DECL_NPY_API(PyArray_FromAny); + DECL_NPY_API(PyArray_NewCopy); + DECL_NPY_API(PyArray_NewFromDescr); +#undef DECL_NPY_API return api; } -- cgit v1.2.3 From a67c2b52e470613620e610e6210687ed084b4949 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:50:06 +0100 Subject: Use memoryview for constructing array from buffer --- include/pybind11/numpy.h | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8e8abe3..19965ba 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -13,6 +13,7 @@ #include "complex.h" #include #include +#include #if defined(_MSC_VER) #pragma warning(push) @@ -31,6 +32,7 @@ public: API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, + API_PyArray_GetArrayParamsFromObject = 278, NPY_C_CONTIGUOUS_ = 0x0001, NPY_F_CONTIGUOUS_ = 0x0002, @@ -61,6 +63,7 @@ public: DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_GetArrayParamsFromObject); #undef DECL_NPY_API return api; } @@ -74,6 +77,8 @@ public: PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); + int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, + Py_ssize_t *, PyObject **, PyObject *); }; PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_) @@ -100,24 +105,22 @@ public: } array(const buffer_info &info) { - API& api = lookup_api(); - if ((info.format.size() < 1) || (info.format.size() > 2)) - pybind11_fail("Unsupported buffer format!"); - int fmt = (int) info.format[0]; - if (info.format == "Zd") fmt = API::NPY_CDOUBLE_; - else if (info.format == "Zf") fmt = API::NPY_CFLOAT_; + PyObject *arr = nullptr, *descr = nullptr; + int ndim = 0; + Py_ssize_t dims[32]; - PyObject *descr = api.PyArray_DescrFromType_(fmt); - if (descr == nullptr) - pybind11_fail("NumPy: unsupported buffer format '" + info.format + "'!"); - object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0], - (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); - if (info.ptr && tmp) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - m_ptr = tmp.release().ptr(); + // allocate zeroed memory if it hasn't been provided + auto buf_info = info; + if (!buf_info.ptr) + buf_info.ptr = std::calloc(info.size, info.itemsize); + auto view = py::memoryview(buf_info); + + API& api = lookup_api(); + auto res = api.PyArray_GetArrayParamsFromObject_(view.ptr(), nullptr, 1, &descr, + &ndim, dims, &arr, nullptr); + if (res < 0 || !arr || descr) + pybind11_fail("NumPy: unable to convert buffer to an array"); + m_ptr = arr; } protected: -- cgit v1.2.3 From fab02efb10d5d3a057eba85752b91d1d77bfcd4b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 14:53:20 +0100 Subject: Switch away from typenums for numpy descriptors --- include/pybind11/numpy.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 19965ba..49489ce 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -91,9 +91,7 @@ public: template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::typenum()); - if (descr == nullptr) - pybind11_fail("NumPy: unsupported buffer format!"); + PyObject *descr = detail::npy_format_descriptor::descr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); @@ -128,6 +126,8 @@ protected: static API api = API::lookup(); return api; } + + template friend struct detail::npy_format_descriptor; }; template class array_t : public array { @@ -140,7 +140,7 @@ public: if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::typenum()); + PyObject *descr = detail::npy_format_descriptor::descr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -158,6 +158,10 @@ private: array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; public: static int typenum() { return values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)]; } + static PyObject* descr() { + if (auto obj = array::lookup_api().PyArray_DescrFromType_(typenum())) return obj; + else pybind11_fail("Unsupported buffer format!"); + } template ::value, int>::type = 0> static PYBIND11_DESCR name() { return _("int") + _(); } template ::value, int>::type = 0> @@ -167,7 +171,11 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - static int typenum() { return array::API::NumPyName; } \ + static int typenum() { return array::API::NumPyName; } \ + static PyObject* descr() { \ + if (auto obj = array::lookup_api().PyArray_DescrFromType_(typenum())) return obj; \ + else pybind11_fail("Unsupported buffer format!"); \ + } \ static PYBIND11_DESCR name() { return _(Name); } } DECL_FMT(float, NPY_FLOAT_, "float32"); DECL_FMT(double, NPY_DOUBLE_, "float64"); -- cgit v1.2.3 From 2488b32066e6a7199719bd1f056037bbffc39b52 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 15:48:55 +0100 Subject: Add PYBIND11_DTYPE macro for registering dtypes --- include/pybind11/numpy.h | 107 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 49489ce..186b82f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #if defined(_MSC_VER) #pragma warning(push) @@ -32,6 +34,7 @@ public: API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, + API_PyArray_DescrConverter = 174, API_PyArray_GetArrayParamsFromObject = 278, NPY_C_CONTIGUOUS_ = 0x0001, @@ -63,6 +66,7 @@ public: DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_GetArrayParamsFromObject); #undef DECL_NPY_API return api; @@ -77,6 +81,7 @@ public: PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); + int (*PyArray_DescrConverter_) (PyObject *, PyObject **); int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, Py_ssize_t *, PyObject **, PyObject *); }; @@ -149,6 +154,19 @@ public: } }; +template struct format_descriptor +::value && + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value>::type> +{ + static const char *value() { + return detail::npy_format_descriptor::format_str(); + } +}; + NAMESPACE_BEGIN(detail) template struct npy_format_descriptor::value>::type> { @@ -184,6 +202,95 @@ DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #undef DECL_FMT +struct field_descriptor { + const char *name; + int offset; + PyObject *descr; +}; + +template struct npy_format_descriptor +::value && // offsetof only works correctly for POD types + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value>::type> +{ + static PYBIND11_DESCR name() { return _("user-defined"); } + + static PyObject* descr() { + if (!descr_()) + pybind11_fail("NumPy: unsupported buffer format!"); + return descr_(); + } + + static const char* format_str() { + return format_str_(); + } + + static void register_dtype(std::initializer_list fields) { + array::API& api = array::lookup_api(); + auto args = py::dict(); + py::list names { }, offsets { }, formats { }; + std::vector dtypes; + for (auto field : fields) { + names.append(py::str(field.name)); + offsets.append(py::int_(field.offset)); + if (!field.descr) + pybind11_fail("NumPy: unsupported field dtype"); + dtypes.emplace_back(field.descr, false); + formats.append(dtypes.back()); + } + args["names"] = names; + args["offsets"] = offsets; + args["formats"] = formats; + if (!api.PyArray_DescrConverter_(args.ptr(), &descr_()) || !descr_()) + pybind11_fail("NumPy: failed to create structured dtype"); + auto np = module::import("numpy"); + auto empty = (object) np.attr("empty"); + if (auto arr = (object) empty(py::int_(0), object(descr(), true))) + if (auto view = PyMemoryView_FromObject(arr.ptr())) + if (auto info = PyMemoryView_GET_BUFFER(view)) { + std::strncpy(format_str_(), info->format, 4096); + return; + } + pybind11_fail("NumPy: failed to extract buffer format"); + } + +private: + static inline PyObject*& descr_() { static PyObject *ptr = nullptr; return ptr; } + static inline char* format_str_() { static char s[4096]; return s; } +}; + +#define FIELD_DESCRIPTOR(Type, Field) \ + ::pybind11::detail::field_descriptor { \ + #Field, offsetof(Type, Field), \ + ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() } + +// The main idea of this macro is borrowed from https://github.com/swansontec/map-macro +// (C) William Swanson, Paul Fultz +#define EVAL0(...) __VA_ARGS__ +#define EVAL1(...) EVAL0 (EVAL0 (EVAL0 (__VA_ARGS__))) +#define EVAL2(...) EVAL1 (EVAL1 (EVAL1 (__VA_ARGS__))) +#define EVAL3(...) EVAL2 (EVAL2 (EVAL2 (__VA_ARGS__))) +#define EVAL4(...) EVAL3 (EVAL3 (EVAL3 (__VA_ARGS__))) +#define EVAL(...) EVAL4 (EVAL4 (EVAL4 (__VA_ARGS__))) +#define MAP_END(...) +#define MAP_OUT +#define MAP_COMMA , +#define MAP_GET_END() 0, MAP_END +#define MAP_NEXT0(test, next, ...) next MAP_OUT +#define MAP_NEXT1(test, next) MAP_NEXT0 (test, next, 0) +#define MAP_NEXT(test, next) MAP_NEXT1 (MAP_GET_END test, next) +#define MAP_LIST_NEXT1(test, next) MAP_NEXT0 (test, MAP_COMMA next, 0) +#define MAP_LIST_NEXT(test, next) MAP_LIST_NEXT1 (MAP_GET_END test, next) +#define MAP_LIST0(f, t, x, peek, ...) f(t, x) MAP_LIST_NEXT (peek, MAP_LIST1) (f, t, peek, __VA_ARGS__) +#define MAP_LIST1(f, t, x, peek, ...) f(t, x) MAP_LIST_NEXT (peek, MAP_LIST0) (f, t, peek, __VA_ARGS__) +#define MAP_LIST(f, t, ...) EVAL (MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) + +#define PYBIND11_DTYPE(Type, ...) \ + ::pybind11::detail::npy_format_descriptor::register_dtype({MAP_LIST(FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + template using array_iterator = typename std::add_pointer::type; -- cgit v1.2.3 From f10c84eb9b91e62a545eb326da90194c502f023b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:04:01 +0100 Subject: Release format descriptor args before converting --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 186b82f..92ab116 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -244,7 +244,7 @@ template struct npy_format_descriptor args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; - if (!api.PyArray_DescrConverter_(args.ptr(), &descr_()) || !descr_()) + if (!api.PyArray_DescrConverter_(args.release().ptr(), &descr_()) || !descr_()) pybind11_fail("NumPy: failed to create structured dtype"); auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); -- cgit v1.2.3 From 2e1565e414bbc34e72a67bfdbf44c2706546d355 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:05:23 +0100 Subject: Add empty recarray test, check for calloc fail --- include/pybind11/numpy.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 92ab116..6e9a629 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -116,6 +116,8 @@ public: auto buf_info = info; if (!buf_info.ptr) buf_info.ptr = std::calloc(info.size, info.itemsize); + if (!buf_info.ptr) + pybind11_fail("NumPy: failed to allocate memory for buffer"); auto view = py::memoryview(buf_info); API& api = lookup_api(); -- cgit v1.2.3 From 80a3785a66a61d4aaa3543a450ef249fbec33d5a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 19 Jun 2016 16:40:52 +0100 Subject: Borrow field descriptors for recarray dtype --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6e9a629..3f538e5 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -240,7 +240,7 @@ template struct npy_format_descriptor offsets.append(py::int_(field.offset)); if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); - dtypes.emplace_back(field.descr, false); + dtypes.emplace_back(field.descr, true); formats.append(dtypes.back()); } args["names"] = names; -- cgit v1.2.3 From f5b166d042b8c1edd9bff6e2f8604e54843d7924 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Tue, 21 Jun 2016 21:03:58 +0100 Subject: Simplify npy_format_descriptor slightly --- include/pybind11/numpy.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3f538e5..1d957d1 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -234,14 +234,12 @@ template struct npy_format_descriptor array::API& api = array::lookup_api(); auto args = py::dict(); py::list names { }, offsets { }, formats { }; - std::vector dtypes; for (auto field : fields) { - names.append(py::str(field.name)); - offsets.append(py::int_(field.offset)); if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); - dtypes.emplace_back(field.descr, true); - formats.append(dtypes.back()); + names.append(py::str(field.name)); + offsets.append(py::int_(field.offset)); + formats.append(object(field.descr, true)); } args["names"] = names; args["offsets"] = offsets; -- cgit v1.2.3 From 2a7acb6d55c0f1e59df1ddb518de7fe6c2fa6456 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 00:33:56 +0100 Subject: Incref descriptors properly when creating arrays --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1d957d1..41d0fea 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -96,7 +96,7 @@ public: template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = detail::npy_format_descriptor::descr(); + PyObject *descr = object(detail::npy_format_descriptor::descr(), true).release().ptr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); @@ -147,7 +147,7 @@ public: if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = detail::npy_format_descriptor::descr(); + PyObject *descr = object(detail::npy_format_descriptor::descr(), true).release().ptr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); -- cgit v1.2.3 From 1f54cd92096b163185fa15669f359e0dc70c1a73 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 00:42:10 +0100 Subject: Use object instead of ptrs in numpy descriptors --- include/pybind11/numpy.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 41d0fea..2889099 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -96,7 +96,7 @@ public: template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = object(detail::npy_format_descriptor::descr(), true).release().ptr(); + PyObject *descr = detail::npy_format_descriptor::descr().release().ptr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); @@ -147,7 +147,7 @@ public: if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = object(detail::npy_format_descriptor::descr(), true).release().ptr(); + PyObject *descr = detail::npy_format_descriptor::descr().release().ptr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -178,8 +178,8 @@ private: array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; public: static int typenum() { return values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)]; } - static PyObject* descr() { - if (auto obj = array::lookup_api().PyArray_DescrFromType_(typenum())) return obj; + static py::object descr() { + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return py::object(ptr, true); else pybind11_fail("Unsupported buffer format!"); } template ::value, int>::type = 0> @@ -192,8 +192,8 @@ template constexpr const int npy_format_descriptor< #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ static int typenum() { return array::API::NumPyName; } \ - static PyObject* descr() { \ - if (auto obj = array::lookup_api().PyArray_DescrFromType_(typenum())) return obj; \ + static py::object descr() { \ + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return py::object(ptr, true); \ else pybind11_fail("Unsupported buffer format!"); \ } \ static PYBIND11_DESCR name() { return _(Name); } } @@ -207,7 +207,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); struct field_descriptor { const char *name; int offset; - PyObject *descr; + py::object descr; }; template struct npy_format_descriptor @@ -220,10 +220,10 @@ template struct npy_format_descriptor { static PYBIND11_DESCR name() { return _("user-defined"); } - static PyObject* descr() { + static py::object descr() { if (!descr_()) pybind11_fail("NumPy: unsupported buffer format!"); - return descr_(); + return py::object(descr_(), true); } static const char* format_str() { @@ -239,7 +239,7 @@ template struct npy_format_descriptor pybind11_fail("NumPy: unsupported field dtype"); names.append(py::str(field.name)); offsets.append(py::int_(field.offset)); - formats.append(object(field.descr, true)); + formats.append(field.descr); } args["names"] = names; args["offsets"] = offsets; @@ -265,7 +265,8 @@ private: #define FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ - ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() } + ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() \ + } // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz -- cgit v1.2.3 From 873d2674716fdfbdf5b7ccf4d6cd951693d9a555 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 00:48:36 +0100 Subject: Prefix all macros in numpy.h to avoid name clashes --- include/pybind11/numpy.h | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2889099..c41b8da 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -270,27 +270,30 @@ private: // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz -#define EVAL0(...) __VA_ARGS__ -#define EVAL1(...) EVAL0 (EVAL0 (EVAL0 (__VA_ARGS__))) -#define EVAL2(...) EVAL1 (EVAL1 (EVAL1 (__VA_ARGS__))) -#define EVAL3(...) EVAL2 (EVAL2 (EVAL2 (__VA_ARGS__))) -#define EVAL4(...) EVAL3 (EVAL3 (EVAL3 (__VA_ARGS__))) -#define EVAL(...) EVAL4 (EVAL4 (EVAL4 (__VA_ARGS__))) -#define MAP_END(...) -#define MAP_OUT -#define MAP_COMMA , -#define MAP_GET_END() 0, MAP_END -#define MAP_NEXT0(test, next, ...) next MAP_OUT -#define MAP_NEXT1(test, next) MAP_NEXT0 (test, next, 0) -#define MAP_NEXT(test, next) MAP_NEXT1 (MAP_GET_END test, next) -#define MAP_LIST_NEXT1(test, next) MAP_NEXT0 (test, MAP_COMMA next, 0) -#define MAP_LIST_NEXT(test, next) MAP_LIST_NEXT1 (MAP_GET_END test, next) -#define MAP_LIST0(f, t, x, peek, ...) f(t, x) MAP_LIST_NEXT (peek, MAP_LIST1) (f, t, peek, __VA_ARGS__) -#define MAP_LIST1(f, t, x, peek, ...) f(t, x) MAP_LIST_NEXT (peek, MAP_LIST0) (f, t, peek, __VA_ARGS__) -#define MAP_LIST(f, t, ...) EVAL (MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) +#define PB11_IMPL_EVAL0(...) __VA_ARGS__ +#define PB11_IMPL_EVAL1(...) PB11_IMPL_EVAL0 (PB11_IMPL_EVAL0 (PB11_IMPL_EVAL0 (__VA_ARGS__))) +#define PB11_IMPL_EVAL2(...) PB11_IMPL_EVAL1 (PB11_IMPL_EVAL1 (PB11_IMPL_EVAL1 (__VA_ARGS__))) +#define PB11_IMPL_EVAL3(...) PB11_IMPL_EVAL2 (PB11_IMPL_EVAL2 (PB11_IMPL_EVAL2 (__VA_ARGS__))) +#define PB11_IMPL_EVAL4(...) PB11_IMPL_EVAL3 (PB11_IMPL_EVAL3 (PB11_IMPL_EVAL3 (__VA_ARGS__))) +#define PB11_IMPL_EVAL(...) PB11_IMPL_EVAL4 (PB11_IMPL_EVAL4 (PB11_IMPL_EVAL4 (__VA_ARGS__))) +#define PB11_IMPL_MAP_END(...) +#define PB11_IMPL_MAP_OUT +#define PB11_IMPL_MAP_COMMA , +#define PB11_IMPL_MAP_GET_END() 0, PB11_IMPL_MAP_END +#define PB11_IMPL_MAP_NEXT0(test, next, ...) next PB11_IMPL_MAP_OUT +#define PB11_IMPL_MAP_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, next, 0) +#define PB11_IMPL_MAP_NEXT(test, next) PB11_IMPL_MAP_NEXT1 (PB11_IMPL_MAP_GET_END test, next) +#define PB11_IMPL_MAP_LIST_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0) +#define PB11_IMPL_MAP_LIST_NEXT(test, next) PB11_IMPL_MAP_LIST_NEXT1 (PB11_IMPL_MAP_GET_END test, next) +#define PB11_IMPL_MAP_LIST0(f, t, x, peek, ...) \ + f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST1) (f, t, peek, __VA_ARGS__) +#define PB11_IMPL_MAP_LIST1(f, t, x, peek, ...) \ + f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST0) (f, t, peek, __VA_ARGS__) +#define PB11_IMPL_MAP_LIST(f, t, ...) PB11_IMPL_EVAL (PB11_IMPL_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) #define PYBIND11_DTYPE(Type, ...) \ - ::pybind11::detail::npy_format_descriptor::register_dtype({MAP_LIST(FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + ::pybind11::detail::npy_format_descriptor::register_dtype \ + ({PB11_IMPL_MAP_LIST(FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) template using array_iterator = typename std::add_pointer::type; -- cgit v1.2.3 From 036e8cd32f067953d7236a4612d9f3e4674650ce Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 00:52:16 +0100 Subject: Remove erroneous py:: prefix in numpy.h --- include/pybind11/numpy.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index c41b8da..1fc6752 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -118,7 +118,7 @@ public: buf_info.ptr = std::calloc(info.size, info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); - auto view = py::memoryview(buf_info); + auto view = memoryview(buf_info); API& api = lookup_api(); auto res = api.PyArray_GetArrayParamsFromObject_(view.ptr(), nullptr, 1, &descr, @@ -178,8 +178,8 @@ private: array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; public: static int typenum() { return values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)]; } - static py::object descr() { - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return py::object(ptr, true); + static object descr() { + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return object(ptr, true); else pybind11_fail("Unsupported buffer format!"); } template ::value, int>::type = 0> @@ -192,8 +192,8 @@ template constexpr const int npy_format_descriptor< #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ static int typenum() { return array::API::NumPyName; } \ - static py::object descr() { \ - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return py::object(ptr, true); \ + static object descr() { \ + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return object(ptr, true); \ else pybind11_fail("Unsupported buffer format!"); \ } \ static PYBIND11_DESCR name() { return _(Name); } } @@ -207,7 +207,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); struct field_descriptor { const char *name; int offset; - py::object descr; + object descr; }; template struct npy_format_descriptor @@ -220,10 +220,10 @@ template struct npy_format_descriptor { static PYBIND11_DESCR name() { return _("user-defined"); } - static py::object descr() { + static object descr() { if (!descr_()) pybind11_fail("NumPy: unsupported buffer format!"); - return py::object(descr_(), true); + return object(descr_(), true); } static const char* format_str() { @@ -232,13 +232,13 @@ template struct npy_format_descriptor static void register_dtype(std::initializer_list fields) { array::API& api = array::lookup_api(); - auto args = py::dict(); - py::list names { }, offsets { }, formats { }; + auto args = dict(); + list names { }, offsets { }, formats { }; for (auto field : fields) { if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); - names.append(py::str(field.name)); - offsets.append(py::int_(field.offset)); + names.append(str(field.name)); + offsets.append(int_(field.offset)); formats.append(field.descr); } args["names"] = names; @@ -248,7 +248,7 @@ template struct npy_format_descriptor pybind11_fail("NumPy: failed to create structured dtype"); auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); - if (auto arr = (object) empty(py::int_(0), object(descr(), true))) + if (auto arr = (object) empty(int_(0), object(descr(), true))) if (auto view = PyMemoryView_FromObject(arr.ptr())) if (auto info = PyMemoryView_GET_BUFFER(view)) { std::strncpy(format_str_(), info->format, 4096); -- cgit v1.2.3 From 4f164217e48ae1dbce5cfb896195c7b56521fc2a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 22 Jun 2016 01:07:20 +0100 Subject: Add dtype_of() function, update the tests --- include/pybind11/numpy.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1fc6752..19ba4de 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -169,6 +169,11 @@ template struct format_descriptor } }; +template +object dtype_of() { + return detail::npy_format_descriptor::descr(); +} + NAMESPACE_BEGIN(detail) template struct npy_format_descriptor::value>::type> { -- cgit v1.2.3 From 5e71e17bdfa13c15e4ac752fec1fa1d7684d15e7 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 12:42:34 +0100 Subject: Make changes to format_descriptor backwards-compat The format strings that are known at compile time are now accessible via both ::value and ::format(), and format strings for everything else is accessible via ::format(). This makes it backwards compatible. --- include/pybind11/numpy.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 19ba4de..cac19ee 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -164,8 +164,8 @@ template struct format_descriptor !std::is_same>::value && !std::is_same>::value>::type> { - static const char *value() { - return detail::npy_format_descriptor::format_str(); + static const char *format() { + return detail::npy_format_descriptor::format(); } }; @@ -231,8 +231,8 @@ template struct npy_format_descriptor return object(descr_(), true); } - static const char* format_str() { - return format_str_(); + static const char* format() { + return format_(); } static void register_dtype(std::initializer_list fields) { @@ -256,7 +256,7 @@ template struct npy_format_descriptor if (auto arr = (object) empty(int_(0), object(descr(), true))) if (auto view = PyMemoryView_FromObject(arr.ptr())) if (auto info = PyMemoryView_GET_BUFFER(view)) { - std::strncpy(format_str_(), info->format, 4096); + std::strncpy(format_(), info->format, 4096); return; } pybind11_fail("NumPy: failed to extract buffer format"); @@ -264,7 +264,7 @@ template struct npy_format_descriptor private: static inline PyObject*& descr_() { static PyObject *ptr = nullptr; return ptr; } - static inline char* format_str_() { static char s[4096]; return s; } + static inline char* format_() { static char s[4096]; return s; } }; #define FIELD_DESCRIPTOR(Type, Field) \ @@ -480,7 +480,7 @@ struct vectorize_helper { return cast(f(*((Args *) buffers[Index].ptr)...)); array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::value(), + format_descriptor::format(), ndim, shape, strides)); buffer_info buf = result.request(); -- cgit v1.2.3 From 95e9b1232262000fe798fe63a5d0fd8b794faf2e Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:18:46 +0100 Subject: Prefix the FIELD_DESCRIPTOR macro --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index cac19ee..aacc993 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -267,7 +267,7 @@ private: static inline char* format_() { static char s[4096]; return s; } }; -#define FIELD_DESCRIPTOR(Type, Field) \ +#define PB11_IMPL_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() \ @@ -298,7 +298,7 @@ private: #define PYBIND11_DTYPE(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ - ({PB11_IMPL_MAP_LIST(FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + ({PB11_IMPL_MAP_LIST(PB11_IMPL_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) template using array_iterator = typename std::add_pointer::type; -- cgit v1.2.3 From 40eadfeb736fa46aa1f5d1932a4bda07712071f4 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:19:18 +0100 Subject: Make npy_format_descriptor backwards-compat The typenum for non-structured types is still accessible at ::value, and the dtype object for all types is accessible at ::dtype(). --- include/pybind11/numpy.h | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index aacc993..e14a58e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -96,7 +96,7 @@ public: template array(size_t size, const Type *ptr) { API& api = lookup_api(); - PyObject *descr = detail::npy_format_descriptor::descr().release().ptr(); + PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); @@ -147,7 +147,7 @@ public: if (ptr == nullptr) return nullptr; API &api = lookup_api(); - PyObject *descr = detail::npy_format_descriptor::descr().release().ptr(); + PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -171,7 +171,7 @@ template struct format_descriptor template object dtype_of() { - return detail::npy_format_descriptor::descr(); + return detail::npy_format_descriptor::dtype(); } NAMESPACE_BEGIN(detail) @@ -182,10 +182,11 @@ private: array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_, array::API::NPY_USHORT_, array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; public: - static int typenum() { return values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)]; } - static object descr() { - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return object(ptr, true); - else pybind11_fail("Unsupported buffer format!"); + enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; + static object dtype() { + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(value)) + return object(ptr, true); + pybind11_fail("Unsupported buffer format!"); } template ::value, int>::type = 0> static PYBIND11_DESCR name() { return _("int") + _(); } @@ -196,10 +197,11 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - static int typenum() { return array::API::NumPyName; } \ - static object descr() { \ - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(typenum())) return object(ptr, true); \ - else pybind11_fail("Unsupported buffer format!"); \ + enum { value = array::API::NumPyName }; \ + static object dtype() { \ + if (auto ptr = array::lookup_api().PyArray_DescrFromType_(value)) \ + return object(ptr, true); \ + pybind11_fail("Unsupported buffer format!"); \ } \ static PYBIND11_DESCR name() { return _(Name); } } DECL_FMT(float, NPY_FLOAT_, "float32"); @@ -225,10 +227,10 @@ template struct npy_format_descriptor { static PYBIND11_DESCR name() { return _("user-defined"); } - static object descr() { - if (!descr_()) + static object dtype() { + if (!dtype_()) pybind11_fail("NumPy: unsupported buffer format!"); - return object(descr_(), true); + return object(dtype_(), true); } static const char* format() { @@ -249,11 +251,11 @@ template struct npy_format_descriptor args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; - if (!api.PyArray_DescrConverter_(args.release().ptr(), &descr_()) || !descr_()) + if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) pybind11_fail("NumPy: failed to create structured dtype"); auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); - if (auto arr = (object) empty(int_(0), object(descr(), true))) + if (auto arr = (object) empty(int_(0), dtype())) if (auto view = PyMemoryView_FromObject(arr.ptr())) if (auto info = PyMemoryView_GET_BUFFER(view)) { std::strncpy(format_(), info->format, 4096); @@ -263,14 +265,14 @@ template struct npy_format_descriptor } private: - static inline PyObject*& descr_() { static PyObject *ptr = nullptr; return ptr; } + static inline PyObject*& dtype_() { static PyObject *ptr = nullptr; return ptr; } static inline char* format_() { static char s[4096]; return s; } }; #define PB11_IMPL_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ - ::pybind11::detail::npy_format_descriptor(0)->Field)>::descr() \ + ::pybind11::detail::npy_format_descriptor(0)->Field)>::dtype() \ } // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro -- cgit v1.2.3 From 5a47a16e47cdbb690b2c1e9337e15d3db6f90fb5 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:21:34 +0100 Subject: Revert accidental whitespace change --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e14a58e..f940852 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -482,7 +482,7 @@ struct vectorize_helper { return cast(f(*((Args *) buffers[Index].ptr)...)); array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::format(), + format_descriptor::format(), ndim, shape, strides)); buffer_info buf = result.request(); -- cgit v1.2.3 From a0e37f250edf86b68ff0e26380883e9a3bf88434 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:34:39 +0100 Subject: npy_format_descriptor::format() - fail if unbound --- include/pybind11/numpy.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index f940852..73cdce0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -234,6 +234,8 @@ template struct npy_format_descriptor } static const char* format() { + if (!dtype_()) + pybind11_fail("NumPy: unsupported buffer format!"); return format_(); } -- cgit v1.2.3 From 73f56830f8e4b024f24717e323d48c5e98fa1275 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 26 Jun 2016 16:46:40 +0100 Subject: Add detail::is_pod_struct helper --- include/pybind11/numpy.h | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 73cdce0..bbd68cf 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -23,7 +23,19 @@ #endif NAMESPACE_BEGIN(pybind11) -namespace detail { template struct npy_format_descriptor { }; } +namespace detail { +template struct npy_format_descriptor { }; + +template +struct is_pod_struct { + enum { value = std::is_pod::value && // offsetof only works correctly for POD types + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value }; +}; +} class array : public buffer { public: @@ -156,14 +168,8 @@ public: } }; -template struct format_descriptor -::value && - !std::is_integral::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same>::value && - !std::is_same>::value>::type> -{ +template +struct format_descriptor::value>::type> { static const char *format() { return detail::npy_format_descriptor::format(); } @@ -217,13 +223,8 @@ struct field_descriptor { object descr; }; -template struct npy_format_descriptor -::value && // offsetof only works correctly for POD types - !std::is_integral::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same>::value && - !std::is_same>::value>::type> +template +struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("user-defined"); } -- cgit v1.2.3 From 5dc6c5445d74bf82f9183fffcb92aafd62ec4a8c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 27 Jun 2016 15:47:51 +0100 Subject: Cosmetic: fix indentation --- include/pybind11/numpy.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index bbd68cf..64fa1dd 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -224,8 +224,7 @@ struct field_descriptor { }; template -struct npy_format_descriptor::value>::type> -{ +struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("user-defined"); } static object dtype() { -- cgit v1.2.3 From 7bdd74a9fbed302a8aa408574b667af3c0fd5bf0 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 27 Jun 2016 17:01:22 +0100 Subject: Fix PYBIND11_DTYPE to work with MSVC compiler --- include/pybind11/numpy.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 64fa1dd..abad9ff 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -292,8 +292,15 @@ private: #define PB11_IMPL_MAP_NEXT0(test, next, ...) next PB11_IMPL_MAP_OUT #define PB11_IMPL_MAP_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, next, 0) #define PB11_IMPL_MAP_NEXT(test, next) PB11_IMPL_MAP_NEXT1 (PB11_IMPL_MAP_GET_END test, next) -#define PB11_IMPL_MAP_LIST_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0) -#define PB11_IMPL_MAP_LIST_NEXT(test, next) PB11_IMPL_MAP_LIST_NEXT1 (PB11_IMPL_MAP_GET_END test, next) +#ifdef _MSC_VER +#define PB11_IMPL_MAP_LIST_NEXT1(test, next) \ + PB11_IMPL_EVAL0 (PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0)) +#else +#define PB11_IMPL_MAP_LIST_NEXT1(test, next) \ + PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0) +#endif +#define PB11_IMPL_MAP_LIST_NEXT(test, next) \ + PB11_IMPL_MAP_LIST_NEXT1 (PB11_IMPL_MAP_GET_END test, next) #define PB11_IMPL_MAP_LIST0(f, t, x, peek, ...) \ f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST1) (f, t, peek, __VA_ARGS__) #define PB11_IMPL_MAP_LIST1(f, t, x, peek, ...) \ -- cgit v1.2.3 From 95545e6256911ea371957f88efdf731628bc402b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 27 Jun 2016 23:02:21 +0100 Subject: Change PB11_IMPL prefix to PYBIND11, add comment --- include/pybind11/numpy.h | 55 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 27 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index abad9ff..dc4000b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -271,7 +271,7 @@ private: static inline char* format_() { static char s[4096]; return s; } }; -#define PB11_IMPL_FIELD_DESCRIPTOR(Type, Field) \ +#define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ ::pybind11::detail::npy_format_descriptor(0)->Field)>::dtype() \ @@ -279,37 +279,38 @@ private: // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz -#define PB11_IMPL_EVAL0(...) __VA_ARGS__ -#define PB11_IMPL_EVAL1(...) PB11_IMPL_EVAL0 (PB11_IMPL_EVAL0 (PB11_IMPL_EVAL0 (__VA_ARGS__))) -#define PB11_IMPL_EVAL2(...) PB11_IMPL_EVAL1 (PB11_IMPL_EVAL1 (PB11_IMPL_EVAL1 (__VA_ARGS__))) -#define PB11_IMPL_EVAL3(...) PB11_IMPL_EVAL2 (PB11_IMPL_EVAL2 (PB11_IMPL_EVAL2 (__VA_ARGS__))) -#define PB11_IMPL_EVAL4(...) PB11_IMPL_EVAL3 (PB11_IMPL_EVAL3 (PB11_IMPL_EVAL3 (__VA_ARGS__))) -#define PB11_IMPL_EVAL(...) PB11_IMPL_EVAL4 (PB11_IMPL_EVAL4 (PB11_IMPL_EVAL4 (__VA_ARGS__))) -#define PB11_IMPL_MAP_END(...) -#define PB11_IMPL_MAP_OUT -#define PB11_IMPL_MAP_COMMA , -#define PB11_IMPL_MAP_GET_END() 0, PB11_IMPL_MAP_END -#define PB11_IMPL_MAP_NEXT0(test, next, ...) next PB11_IMPL_MAP_OUT -#define PB11_IMPL_MAP_NEXT1(test, next) PB11_IMPL_MAP_NEXT0 (test, next, 0) -#define PB11_IMPL_MAP_NEXT(test, next) PB11_IMPL_MAP_NEXT1 (PB11_IMPL_MAP_GET_END test, next) -#ifdef _MSC_VER -#define PB11_IMPL_MAP_LIST_NEXT1(test, next) \ - PB11_IMPL_EVAL0 (PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0)) +#define PYBIND11_EVAL0(...) __VA_ARGS__ +#define PYBIND11_EVAL1(...) PYBIND11_EVAL0 (PYBIND11_EVAL0 (PYBIND11_EVAL0 (__VA_ARGS__))) +#define PYBIND11_EVAL2(...) PYBIND11_EVAL1 (PYBIND11_EVAL1 (PYBIND11_EVAL1 (__VA_ARGS__))) +#define PYBIND11_EVAL3(...) PYBIND11_EVAL2 (PYBIND11_EVAL2 (PYBIND11_EVAL2 (__VA_ARGS__))) +#define PYBIND11_EVAL4(...) PYBIND11_EVAL3 (PYBIND11_EVAL3 (PYBIND11_EVAL3 (__VA_ARGS__))) +#define PYBIND11_EVAL(...) PYBIND11_EVAL4 (PYBIND11_EVAL4 (PYBIND11_EVAL4 (__VA_ARGS__))) +#define PYBIND11_MAP_END(...) +#define PYBIND11_MAP_OUT +#define PYBIND11_MAP_COMMA , +#define PYBIND11_MAP_GET_END() 0, PYBIND11_MAP_END +#define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT +#define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0 (test, next, 0) +#define PYBIND11_MAP_NEXT(test, next) PYBIND11_MAP_NEXT1 (PYBIND11_MAP_GET_END test, next) +#ifdef _MSC_VER // MSVC is not as eager to expand macros +#define PYBIND11_MAP_LIST_NEXT1(test, next) \ + PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)) #else -#define PB11_IMPL_MAP_LIST_NEXT1(test, next) \ - PB11_IMPL_MAP_NEXT0 (test, PB11_IMPL_MAP_COMMA next, 0) +#define PYBIND11_MAP_LIST_NEXT1(test, next) \ + PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0) #endif -#define PB11_IMPL_MAP_LIST_NEXT(test, next) \ - PB11_IMPL_MAP_LIST_NEXT1 (PB11_IMPL_MAP_GET_END test, next) -#define PB11_IMPL_MAP_LIST0(f, t, x, peek, ...) \ - f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST1) (f, t, peek, __VA_ARGS__) -#define PB11_IMPL_MAP_LIST1(f, t, x, peek, ...) \ - f(t, x) PB11_IMPL_MAP_LIST_NEXT (peek, PB11_IMPL_MAP_LIST0) (f, t, peek, __VA_ARGS__) -#define PB11_IMPL_MAP_LIST(f, t, ...) PB11_IMPL_EVAL (PB11_IMPL_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) +#define PYBIND11_MAP_LIST_NEXT(test, next) \ + PYBIND11_MAP_LIST_NEXT1 (PYBIND11_MAP_GET_END test, next) +#define PYBIND11_MAP_LIST0(f, t, x, peek, ...) \ + f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST1) (f, t, peek, __VA_ARGS__) +#define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \ + f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST0) (f, t, peek, __VA_ARGS__) +#define PYBIND11_MAP_LIST(f, t, ...) \ + PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) #define PYBIND11_DTYPE(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ - ({PB11_IMPL_MAP_LIST(PB11_IMPL_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) template using array_iterator = typename std::add_pointer::type; -- cgit v1.2.3 From b38ca22e947f7cc7fcb58297e1f4ce2d36f2d52a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 29 Jun 2016 15:16:49 +0100 Subject: Add a few braces for clarity --- include/pybind11/numpy.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index dc4000b..5fdbe6d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -257,12 +257,14 @@ struct npy_format_descriptor::value> pybind11_fail("NumPy: failed to create structured dtype"); auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); - if (auto arr = (object) empty(int_(0), dtype())) - if (auto view = PyMemoryView_FromObject(arr.ptr())) + if (auto arr = (object) empty(int_(0), dtype())) { + if (auto view = PyMemoryView_FromObject(arr.ptr())) { if (auto info = PyMemoryView_GET_BUFFER(view)) { std::strncpy(format_(), info->format, 4096); return; } + } + } pybind11_fail("NumPy: failed to extract buffer format"); } -- cgit v1.2.3 From 2f01f01866a1efe137e0dbd7160165bca44d087e Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 29 Jun 2016 15:21:10 +0100 Subject: Always allocate at least one element --- include/pybind11/numpy.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5fdbe6d..43f0d57 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -127,7 +127,8 @@ public: // allocate zeroed memory if it hasn't been provided auto buf_info = info; if (!buf_info.ptr) - buf_info.ptr = std::calloc(info.size, info.itemsize); + // always allocate at least 1 element, same way as NumPy does it + buf_info.ptr = std::calloc(std::max(info.size, 1ul), info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); auto view = memoryview(buf_info); -- cgit v1.2.3 From 3b803846d50783f4cb4fe2e41d76b0f9c9f520ea Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 29 Jun 2016 15:21:51 +0100 Subject: Add a few comments throughout numpy.h --- include/pybind11/numpy.h | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 43f0d57..a3bab98 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -123,20 +123,29 @@ public: PyObject *arr = nullptr, *descr = nullptr; int ndim = 0; Py_ssize_t dims[32]; + API& api = lookup_api(); - // allocate zeroed memory if it hasn't been provided + // Allocate zeroed memory if it hasn't been provided by the caller. + // Normally, we could leave this null for NumPy to allocate memory for us, but + // since we need a memoryview, the data pointer has to be non-null. NumPy uses + // malloc if NPY_NEEDS_INIT is not set (in which case it uses calloc); however, + // we don't have a descriptor yet (only a buffer format string), so we can't + // access the flags. The safest thing to do is thus to use calloc. auto buf_info = info; if (!buf_info.ptr) // always allocate at least 1 element, same way as NumPy does it buf_info.ptr = std::calloc(std::max(info.size, 1ul), info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); - auto view = memoryview(buf_info); - API& api = lookup_api(); + // PyArray_GetArrayParamsFromObject seems to be the only low-level API function + // that will accept arbitrary buffers (including structured types) + auto view = memoryview(buf_info); auto res = api.PyArray_GetArrayParamsFromObject_(view.ptr(), nullptr, 1, &descr, &ndim, dims, &arr, nullptr); if (res < 0 || !arr || descr) + // We expect arr to have a pointer to a newly created array, in which case all + // other parameters like descr would be set to null, according to the API. pybind11_fail("NumPy: unable to convert buffer to an array"); m_ptr = arr; } @@ -161,7 +170,8 @@ public: return nullptr; API &api = lookup_api(); PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); - PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, + API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); Py_DECREF(ptr); @@ -254,8 +264,11 @@ struct npy_format_descriptor::value> args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; + // This is essentially the same as calling np.dtype() constructor in Python and passing + // it a dict of the form {'names': ..., 'formats': ..., 'offsets': ...}. if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) pybind11_fail("NumPy: failed to create structured dtype"); + // Let NumPy figure the buffer format string for us: memoryview(np.empty(0, dtype)).format auto np = module::import("numpy"); auto empty = (object) np.attr("empty"); if (auto arr = (object) empty(int_(0), dtype())) { @@ -274,6 +287,7 @@ private: static inline char* format_() { static char s[4096]; return s; } }; +// Extract name, offset and format descriptor for a struct field #define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ #Field, offsetof(Type, Field), \ @@ -295,7 +309,7 @@ private: #define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT #define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0 (test, next, 0) #define PYBIND11_MAP_NEXT(test, next) PYBIND11_MAP_NEXT1 (PYBIND11_MAP_GET_END test, next) -#ifdef _MSC_VER // MSVC is not as eager to expand macros +#ifdef _MSC_VER // MSVC is not as eager to expand macros, hence this workaround #define PYBIND11_MAP_LIST_NEXT1(test, next) \ PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)) #else @@ -308,6 +322,7 @@ private: f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST1) (f, t, peek, __VA_ARGS__) #define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \ f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST0) (f, t, peek, __VA_ARGS__) +// PYBIND11_MAP_LIST(f, t, a1, a2, ...) expands to f(t, a1), f(t, a2), ... #define PYBIND11_MAP_LIST(f, t, ...) \ PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) -- cgit v1.2.3 From 4c9a160a1d85be98c7111f50daa2f41e5fb5cd70 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 29 Jun 2016 15:27:21 +0100 Subject: Exclude double type from is_pod_struct --- include/pybind11/numpy.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a3bab98..393c887 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -31,6 +31,7 @@ struct is_pod_struct { enum { value = std::is_pod::value && // offsetof only works correctly for POD types !std::is_integral::value && !std::is_same::value && + !std::is_same::value && !std::is_same::value && !std::is_same>::value && !std::is_same>::value }; -- cgit v1.2.3 From 872bd92575299cb701823dfe098479915d4e854a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 2 Jul 2016 16:14:57 +0100 Subject: Use proper type for an int literal --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 393c887..6375e16 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -135,7 +135,7 @@ public: auto buf_info = info; if (!buf_info.ptr) // always allocate at least 1 element, same way as NumPy does it - buf_info.ptr = std::calloc(std::max(info.size, 1ul), info.itemsize); + buf_info.ptr = std::calloc(std::max(info.size, (size_t) 1), info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); -- cgit v1.2.3 From 5412a05cf03dc6472456c6dbc2497d764c952d90 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 2 Jul 2016 16:18:42 +0100 Subject: Rename PYBIND11_DTYPE to PYBIND11_NUMPY_DTYPE --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6375e16..c8854ea 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -327,7 +327,7 @@ private: #define PYBIND11_MAP_LIST(f, t, ...) \ PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) -#define PYBIND11_DTYPE(Type, ...) \ +#define PYBIND11_NUMPY_DTYPE(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) -- cgit v1.2.3 From 511401599c4e655675efa75b0b0bf92e219bdda2 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 2 Jul 2016 16:43:21 +0100 Subject: Use malloc insterad of calloc for numpy arrays --- include/pybind11/numpy.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index c8854ea..2d1082f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -126,16 +126,17 @@ public: Py_ssize_t dims[32]; API& api = lookup_api(); - // Allocate zeroed memory if it hasn't been provided by the caller. + // Allocate non-zeroed memory if it hasn't been provided by the caller. // Normally, we could leave this null for NumPy to allocate memory for us, but // since we need a memoryview, the data pointer has to be non-null. NumPy uses // malloc if NPY_NEEDS_INIT is not set (in which case it uses calloc); however, - // we don't have a descriptor yet (only a buffer format string), so we can't - // access the flags. The safest thing to do is thus to use calloc. + // we don't have a desriptor yet (only a buffer format string), so we can't + // access the flags. As long as we're not dealing with object dtypes/fields + // though, the memory doesn't have to be zeroed so we use malloc. auto buf_info = info; if (!buf_info.ptr) // always allocate at least 1 element, same way as NumPy does it - buf_info.ptr = std::calloc(std::max(info.size, (size_t) 1), info.itemsize); + buf_info.ptr = std::malloc(std::max(info.size, (size_t) 1) * info.itemsize); if (!buf_info.ptr) pybind11_fail("NumPy: failed to allocate memory for buffer"); -- cgit v1.2.3 From eeb4c043f9b9fbc9a270910101ed2f45d823533b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 3 Jul 2016 10:22:10 +0100 Subject: Change field descriptor offset type to size_t --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2d1082f..1046cd4 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -232,7 +232,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); struct field_descriptor { const char *name; - int offset; + size_t offset; object descr; }; -- cgit v1.2.3 From 13022f1b8cf533711994b505858025fcd3ea345c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 3 Jul 2016 10:22:40 +0100 Subject: Bugfix: pass struct size as itemsize to descriptor Without this, partially bound structs will have incorrect itemsize. --- include/pybind11/numpy.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1046cd4..423403d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -266,6 +266,7 @@ struct npy_format_descriptor::value> args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; + args["itemsize"] = int_(sizeof(T)); // This is essentially the same as calling np.dtype() constructor in Python and passing // it a dict of the form {'names': ..., 'formats': ..., 'offsets': ...}. if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) -- cgit v1.2.3 From 8fa09cb871d2241355bb2a794dc75db502b20997 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 6 Jul 2016 00:28:12 +0100 Subject: Strip padding fields in dtypes, update the tests --- include/pybind11/numpy.h | 165 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 116 insertions(+), 49 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 423403d..b8827c3 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #if defined(_MSC_VER) @@ -26,6 +27,8 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; +object fix_dtype(object); + template struct is_pod_struct { enum { value = std::is_pod::value && // offsetof only works correctly for POD types @@ -47,7 +50,9 @@ public: API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, + API_PyArray_DescrNewFromType = 9, API_PyArray_DescrConverter = 174, + API_PyArray_EquivTypes = 182, API_PyArray_GetArrayParamsFromObject = 278, NPY_C_CONTIGUOUS_ = 0x0001, @@ -61,7 +66,9 @@ public: NPY_LONG_, NPY_ULONG_, NPY_LONGLONG_, NPY_ULONGLONG_, NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, - NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_ + NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, + NPY_OBJECT_ = 17, + NPY_STRING_, NPY_UNICODE_, NPY_VOID_ }; static API lookup() { @@ -79,7 +86,9 @@ public: DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_DescrNewFromType); DECL_NPY_API(PyArray_DescrConverter); + DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_GetArrayParamsFromObject); #undef DECL_NPY_API return api; @@ -91,10 +100,12 @@ public: PyObject *(*PyArray_NewFromDescr_) (PyTypeObject *, PyObject *, int, Py_intptr_t *, Py_intptr_t *, void *, int, PyObject *); + PyObject *(*PyArray_DescrNewFromType_)(int); PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); int (*PyArray_DescrConverter_) (PyObject *, PyObject **); + bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, Py_ssize_t *, PyObject **, PyObject *); }; @@ -113,52 +124,83 @@ public: Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); - if (ptr && tmp) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); + if (ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); m_ptr = tmp.release().ptr(); } array(const buffer_info &info) { - PyObject *arr = nullptr, *descr = nullptr; - int ndim = 0; - Py_ssize_t dims[32]; - API& api = lookup_api(); - - // Allocate non-zeroed memory if it hasn't been provided by the caller. - // Normally, we could leave this null for NumPy to allocate memory for us, but - // since we need a memoryview, the data pointer has to be non-null. NumPy uses - // malloc if NPY_NEEDS_INIT is not set (in which case it uses calloc); however, - // we don't have a desriptor yet (only a buffer format string), so we can't - // access the flags. As long as we're not dealing with object dtypes/fields - // though, the memory doesn't have to be zeroed so we use malloc. - auto buf_info = info; - if (!buf_info.ptr) - // always allocate at least 1 element, same way as NumPy does it - buf_info.ptr = std::malloc(std::max(info.size, (size_t) 1) * info.itemsize); - if (!buf_info.ptr) - pybind11_fail("NumPy: failed to allocate memory for buffer"); - - // PyArray_GetArrayParamsFromObject seems to be the only low-level API function - // that will accept arbitrary buffers (including structured types) - auto view = memoryview(buf_info); - auto res = api.PyArray_GetArrayParamsFromObject_(view.ptr(), nullptr, 1, &descr, - &ndim, dims, &arr, nullptr); - if (res < 0 || !arr || descr) - // We expect arr to have a pointer to a newly created array, in which case all - // other parameters like descr would be set to null, according to the API. - pybind11_fail("NumPy: unable to convert buffer to an array"); - m_ptr = arr; + auto& api = lookup_api(); + + // _dtype_from_pep3118 returns dtypes with padding fields in, however the array + // constructor seems to then consume them, so we don't need to strip them ourselves + auto numpy_internal = module::import("numpy.core._internal"); + auto dtype_from_fmt = (object) numpy_internal.attr("_dtype_from_pep3118"); + auto dtype = dtype_from_fmt(pybind11::str(info.format)); + auto dtype2 = strip_padding_fields(dtype); + + object tmp(api.PyArray_NewFromDescr_( + api.PyArray_Type_, dtype2.release().ptr(), (int) info.ndim, (Py_intptr_t *) &info.shape[0], + (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); + if (!tmp) + pybind11_fail("NumPy: unable to create array!"); + if (info.ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + m_ptr = tmp.release().ptr(); + auto d = (object) this->attr("dtype"); } -protected: +// protected: static API &lookup_api() { static API api = API::lookup(); return api; } template friend struct detail::npy_format_descriptor; + + static object strip_padding_fields(object dtype) { + // Recursively strip all void fields with empty names that are generated for + // padding fields (as of NumPy v1.11). + auto fields = dtype.attr("fields").cast(); + if (fields.ptr() == Py_None) + return dtype; + + struct field_descr { pybind11::str name; object format; int_ offset; }; + std::vector field_descriptors; + + auto items = fields.attr("items").cast(); + for (auto field : items()) { + auto spec = object(field, true).cast(); + auto name = spec[0].cast(); + auto format = spec[1].cast()[0].cast(); + auto offset = spec[1].cast()[1].cast(); + if (!len(name) && (std::string) dtype.attr("kind").cast() == "V") + continue; + field_descriptors.push_back({name, strip_padding_fields(format), offset}); + } + + std::sort(field_descriptors.begin(), field_descriptors.end(), + [](const field_descr& a, const field_descr& b) { + return (int) a.offset < (int) b.offset; + }); + + list names, formats, offsets; + for (auto& descr : field_descriptors) { + names.append(descr.name); + formats.append(descr.format); + offsets.append(descr.offset); + } + auto args = dict(); + args["names"] = names; args["formats"] = formats; args["offsets"] = offsets; + args["itemsize"] = dtype.attr("itemsize").cast(); + + PyObject *descr = nullptr; + if (!lookup_api().PyArray_DescrConverter_(args.release().ptr(), &descr) || !descr) + pybind11_fail("NumPy: failed to create structured dtype"); + return object(descr, false); + } }; template class array_t : public array { @@ -233,9 +275,12 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); struct field_descriptor { const char *name; size_t offset; + size_t size; + const char *format; object descr; }; + template struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("user-defined"); } @@ -253,7 +298,7 @@ struct npy_format_descriptor::value> } static void register_dtype(std::initializer_list fields) { - array::API& api = array::lookup_api(); + auto& api = array::lookup_api(); auto args = dict(); list names { }, offsets { }, formats { }; for (auto field : fields) { @@ -263,26 +308,47 @@ struct npy_format_descriptor::value> offsets.append(int_(field.offset)); formats.append(field.descr); } - args["names"] = names; - args["offsets"] = offsets; - args["formats"] = formats; + args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; args["itemsize"] = int_(sizeof(T)); // This is essentially the same as calling np.dtype() constructor in Python and passing // it a dict of the form {'names': ..., 'formats': ..., 'offsets': ...}. if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) pybind11_fail("NumPy: failed to create structured dtype"); - // Let NumPy figure the buffer format string for us: memoryview(np.empty(0, dtype)).format - auto np = module::import("numpy"); - auto empty = (object) np.attr("empty"); - if (auto arr = (object) empty(int_(0), dtype())) { - if (auto view = PyMemoryView_FromObject(arr.ptr())) { - if (auto info = PyMemoryView_GET_BUFFER(view)) { - std::strncpy(format_(), info->format, 4096); - return; - } - } + + // There is an existing bug in NumPy (as of v1.11): trailing bytes are + // not encoded explicitly into the format string. This will supposedly + // get fixed in v1.12; for further details, see these: + // - https://github.com/numpy/numpy/issues/7797 + // - https://github.com/numpy/numpy/pull/7798 + // Because of this, we won't use numpy's logic to generate buffer format + // strings and will just do it ourselves. + std::vector ordered_fields(fields); + std::sort(ordered_fields.begin(), ordered_fields.end(), + [](const field_descriptor& a, const field_descriptor &b) { + return a.offset < b.offset; + }); + size_t offset = 0; + std::ostringstream oss; + oss << "T{"; + for (auto& field : ordered_fields) { + if (field.offset > offset) + oss << (field.offset - offset) << 'x'; + // note that '=' is required to cover the case of unaligned fields + oss << '=' << field.format << ':' << field.name << ':'; + offset = field.offset + field.size; } - pybind11_fail("NumPy: failed to extract buffer format"); + if (sizeof(T) > offset) + oss << (sizeof(T) - offset) << 'x'; + oss << '}'; + std::strncpy(format_(), oss.str().c_str(), 4096); + + // Sanity check: verify that NumPy properly parses our buffer format string + auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); + auto dtype = (object) arr.attr("dtype"); + auto fixed_dtype = dtype; + // auto fixed_dtype = array::strip_padding_fields(object(dtype_(), true)); + // if (!api.PyArray_EquivTypes_(dtype_(), fixed_dtype.ptr())) + // pybind11_fail("NumPy: invalid buffer descriptor!"); } private: @@ -293,7 +359,8 @@ private: // Extract name, offset and format descriptor for a struct field #define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ - #Field, offsetof(Type, Field), \ + #Field, offsetof(Type, Field), sizeof(decltype(static_cast(0)->Field)), \ + ::pybind11::format_descriptor(0)->Field)>::format(), \ ::pybind11::detail::npy_format_descriptor(0)->Field)>::dtype() \ } -- cgit v1.2.3 From 8f2f7cd61cae57680baa3cef2c00c307f51b5146 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 17 Jul 2016 11:07:49 +0100 Subject: Various cleanup --- include/pybind11/numpy.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b8827c3..e756db8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -138,21 +138,19 @@ public: // constructor seems to then consume them, so we don't need to strip them ourselves auto numpy_internal = module::import("numpy.core._internal"); auto dtype_from_fmt = (object) numpy_internal.attr("_dtype_from_pep3118"); - auto dtype = dtype_from_fmt(pybind11::str(info.format)); - auto dtype2 = strip_padding_fields(dtype); + auto dtype = strip_padding_fields(dtype_from_fmt(pybind11::str(info.format))); object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, dtype2.release().ptr(), (int) info.ndim, (Py_intptr_t *) &info.shape[0], + api.PyArray_Type_, dtype.release().ptr(), (int) info.ndim, (Py_intptr_t *) &info.shape[0], (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); if (info.ptr) tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); m_ptr = tmp.release().ptr(); - auto d = (object) this->attr("dtype"); } -// protected: +protected: static API &lookup_api() { static API api = API::lookup(); return api; -- cgit v1.2.3 From 076b953ccd6a0849af652755feb7e1d1d19296dc Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 17 Jul 2016 11:10:19 +0100 Subject: Restore dtype equivalence sanity check --- include/pybind11/numpy.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e756db8..58d9695 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -342,11 +342,9 @@ struct npy_format_descriptor::value> // Sanity check: verify that NumPy properly parses our buffer format string auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); - auto dtype = (object) arr.attr("dtype"); - auto fixed_dtype = dtype; - // auto fixed_dtype = array::strip_padding_fields(object(dtype_(), true)); - // if (!api.PyArray_EquivTypes_(dtype_(), fixed_dtype.ptr())) - // pybind11_fail("NumPy: invalid buffer descriptor!"); + auto fixed_dtype = array::strip_padding_fields(object(dtype_(), true)); + if (!api.PyArray_EquivTypes_(dtype_(), fixed_dtype.ptr())) + pybind11_fail("NumPy: invalid buffer descriptor!"); } private: -- cgit v1.2.3 From 41c339902123b96ce3bf23f2522758f5d4954be8 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 19:58:20 +0100 Subject: Update npy_format_descriptor::name() --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 58d9695..030787b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -281,7 +281,7 @@ struct field_descriptor { template struct npy_format_descriptor::value>::type> { - static PYBIND11_DESCR name() { return _("user-defined"); } + static PYBIND11_DESCR name() { return _("struct"); } static object dtype() { if (!dtype_()) -- cgit v1.2.3 From b37985ee0c2c46e689e43542649760872f1a996c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 22:37:42 +0100 Subject: Fix a comment and wrong indentation --- include/pybind11/numpy.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 030787b..12073ba 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -134,8 +134,7 @@ public: array(const buffer_info &info) { auto& api = lookup_api(); - // _dtype_from_pep3118 returns dtypes with padding fields in, however the array - // constructor seems to then consume them, so we don't need to strip them ourselves + // _dtype_from_pep3118 returns dtypes with padding fields in, so we need to strip them auto numpy_internal = module::import("numpy.core._internal"); auto dtype_from_fmt = (object) numpy_internal.attr("_dtype_from_pep3118"); auto dtype = strip_padding_fields(dtype_from_fmt(pybind11::str(info.format))); @@ -175,7 +174,7 @@ protected: auto format = spec[1].cast()[0].cast(); auto offset = spec[1].cast()[1].cast(); if (!len(name) && (std::string) dtype.attr("kind").cast() == "V") - continue; + continue; field_descriptors.push_back({name, strip_padding_fields(format), offset}); } -- cgit v1.2.3 From 098f9aef7382f483128919c97a74c81359a7c911 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 22:52:08 +0100 Subject: Replace 4096B format buffer with std::string --- include/pybind11/numpy.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 12073ba..10eb895 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -291,7 +291,7 @@ struct npy_format_descriptor::value> static const char* format() { if (!dtype_()) pybind11_fail("NumPy: unsupported buffer format!"); - return format_(); + return format_().c_str(); } static void register_dtype(std::initializer_list fields) { @@ -337,7 +337,7 @@ struct npy_format_descriptor::value> if (sizeof(T) > offset) oss << (sizeof(T) - offset) << 'x'; oss << '}'; - std::strncpy(format_(), oss.str().c_str(), 4096); + format_() = oss.str(); // Sanity check: verify that NumPy properly parses our buffer format string auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); @@ -348,7 +348,7 @@ struct npy_format_descriptor::value> private: static inline PyObject*& dtype_() { static PyObject *ptr = nullptr; return ptr; } - static inline char* format_() { static char s[4096]; return s; } + static inline std::string& format_() { static std::string s; return s; } }; // Extract name, offset and format descriptor for a struct field -- cgit v1.2.3 From 103d5eadc36ccc361474c7793bf4b588b16fce23 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 18 Jul 2016 23:36:18 +0100 Subject: Remove redundant definition --- include/pybind11/numpy.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 10eb895..1dc3de2 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -27,7 +27,6 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; -object fix_dtype(object); template struct is_pod_struct { -- cgit v1.2.3 From f9c0defed7a764b1b1cfc02bce6c742d56e2664f Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 20 Jul 2016 00:19:24 +0100 Subject: Add numpy wrappers for char[] and std::array --- include/pybind11/numpy.h | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1dc3de2..20fbfd8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -13,6 +13,7 @@ #include "complex.h" #include #include +#include #include #include #include @@ -27,10 +28,14 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; +template struct is_std_array : std::false_type { }; +template struct is_std_array> : std::true_type { }; template struct is_pod_struct { enum { value = std::is_pod::value && // offsetof only works correctly for POD types + !std::is_array::value && + !is_std_array::value && !std::is_integral::value && !std::is_same::value && !std::is_same::value && @@ -221,9 +226,14 @@ public: template struct format_descriptor::value>::type> { - static const char *format() { - return detail::npy_format_descriptor::format(); - } + static const char *format() { return detail::npy_format_descriptor::format(); } +}; + +template struct format_descriptor { + static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } +}; +template struct format_descriptor> { + static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } }; template @@ -268,6 +278,22 @@ DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #undef DECL_FMT +#define DECL_CHAR_FMT \ + static PYBIND11_DESCR name() { return _("S") + _(); } \ + static object dtype() { \ + auto& api = array::lookup_api(); \ + PyObject *descr = nullptr; \ + PYBIND11_DESCR fmt = _("S") + _(); \ + pybind11::str py_fmt(fmt.text()); \ + if (!api.PyArray_DescrConverter_(py_fmt.release().ptr(), &descr) || !descr) \ + pybind11_fail("NumPy: failed to create string dtype"); \ + return object(descr, false); \ + } \ + static const char *format() { PYBIND11_DESCR s = _() + _("s"); return s.text(); } +template struct npy_format_descriptor { DECL_CHAR_FMT }; +template struct npy_format_descriptor> { DECL_CHAR_FMT }; +#undef DECL_CHAR_FMT + struct field_descriptor { const char *name; size_t offset; -- cgit v1.2.3 From afb07e7e9268e0ef316c6dc58dca30599e878b6f Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 20 Jul 2016 00:26:18 +0100 Subject: Code reordering / cleanup only --- include/pybind11/numpy.h | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 20fbfd8..d0cf1f6 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -27,22 +27,7 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; - -template struct is_std_array : std::false_type { }; -template struct is_std_array> : std::true_type { }; - -template -struct is_pod_struct { - enum { value = std::is_pod::value && // offsetof only works correctly for POD types - !std::is_array::value && - !is_std_array::value && - !std::is_integral::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same>::value && - !std::is_same>::value }; -}; +template struct is_pod_struct; } class array : public buffer { @@ -242,6 +227,21 @@ object dtype_of() { } NAMESPACE_BEGIN(detail) +template struct is_std_array : std::false_type { }; +template struct is_std_array> : std::true_type { }; + +template +struct is_pod_struct { + enum { value = std::is_pod::value && // offsetof only works correctly for POD types + !std::is_array::value && + !is_std_array::value && + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value }; +}; template struct npy_format_descriptor::value>::type> { private: @@ -302,7 +302,6 @@ struct field_descriptor { object descr; }; - template struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("struct"); } -- cgit v1.2.3 From 05cb58ade27ac74979efd7a64cb8d120744521dd Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 20 Jul 2016 00:54:57 +0100 Subject: Cleanup: move numpy API bindings out of py::array --- include/pybind11/numpy.h | 172 ++++++++++++++++++++++++----------------------- 1 file changed, 87 insertions(+), 85 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d0cf1f6..77f7e6f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -28,87 +28,94 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; template struct is_pod_struct; -} -class array : public buffer { -public: - struct API { - enum Entries { - API_PyArray_Type = 2, - API_PyArray_DescrFromType = 45, - API_PyArray_FromAny = 69, - API_PyArray_NewCopy = 85, - API_PyArray_NewFromDescr = 94, - API_PyArray_DescrNewFromType = 9, - API_PyArray_DescrConverter = 174, - API_PyArray_EquivTypes = 182, - API_PyArray_GetArrayParamsFromObject = 278, - - NPY_C_CONTIGUOUS_ = 0x0001, - NPY_F_CONTIGUOUS_ = 0x0002, - NPY_ARRAY_FORCECAST_ = 0x0010, - NPY_ENSURE_ARRAY_ = 0x0040, - NPY_BOOL_ = 0, - NPY_BYTE_, NPY_UBYTE_, - NPY_SHORT_, NPY_USHORT_, - NPY_INT_, NPY_UINT_, - NPY_LONG_, NPY_ULONG_, - NPY_LONGLONG_, NPY_ULONGLONG_, - NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, - NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, - NPY_OBJECT_ = 17, - NPY_STRING_, NPY_UNICODE_, NPY_VOID_ - }; - - static API lookup() { - module m = module::import("numpy.core.multiarray"); - object c = (object) m.attr("_ARRAY_API"); +struct npy_api { + enum constants { + NPY_C_CONTIGUOUS_ = 0x0001, + NPY_F_CONTIGUOUS_ = 0x0002, + NPY_ARRAY_FORCECAST_ = 0x0010, + NPY_ENSURE_ARRAY_ = 0x0040, + NPY_BOOL_ = 0, + NPY_BYTE_, NPY_UBYTE_, + NPY_SHORT_, NPY_USHORT_, + NPY_INT_, NPY_UINT_, + NPY_LONG_, NPY_ULONG_, + NPY_LONGLONG_, NPY_ULONGLONG_, + NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, + NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, + NPY_OBJECT_ = 17, + NPY_STRING_, NPY_UNICODE_, NPY_VOID_ + }; + + static npy_api& get() { + static npy_api api = lookup(); + return api; + } + + bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); } + + PyObject *(*PyArray_DescrFromType_)(int); + PyObject *(*PyArray_NewFromDescr_) + (PyTypeObject *, PyObject *, int, Py_intptr_t *, + Py_intptr_t *, void *, int, PyObject *); + PyObject *(*PyArray_DescrNewFromType_)(int); + PyObject *(*PyArray_NewCopy_)(PyObject *, int); + PyTypeObject *PyArray_Type_; + PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); + int (*PyArray_DescrConverter_) (PyObject *, PyObject **); + bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); + int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, + Py_ssize_t *, PyObject **, PyObject *); +private: + enum functions { + API_PyArray_Type = 2, + API_PyArray_DescrFromType = 45, + API_PyArray_FromAny = 69, + API_PyArray_NewCopy = 85, + API_PyArray_NewFromDescr = 94, + API_PyArray_DescrNewFromType = 9, + API_PyArray_DescrConverter = 174, + API_PyArray_EquivTypes = 182, + API_PyArray_GetArrayParamsFromObject = 278, + }; + + static npy_api lookup() { + module m = module::import("numpy.core.multiarray"); + object c = (object) m.attr("_ARRAY_API"); #if PY_MAJOR_VERSION >= 3 - void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr); + void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr); #else - void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); + void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); #endif - API api; + npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; - DECL_NPY_API(PyArray_Type); - DECL_NPY_API(PyArray_DescrFromType); - DECL_NPY_API(PyArray_FromAny); - DECL_NPY_API(PyArray_NewCopy); - DECL_NPY_API(PyArray_NewFromDescr); - DECL_NPY_API(PyArray_DescrNewFromType); - DECL_NPY_API(PyArray_DescrConverter); - DECL_NPY_API(PyArray_EquivTypes); - DECL_NPY_API(PyArray_GetArrayParamsFromObject); + DECL_NPY_API(PyArray_Type); + DECL_NPY_API(PyArray_DescrFromType); + DECL_NPY_API(PyArray_FromAny); + DECL_NPY_API(PyArray_NewCopy); + DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_DescrNewFromType); + DECL_NPY_API(PyArray_DescrConverter); + DECL_NPY_API(PyArray_EquivTypes); + DECL_NPY_API(PyArray_GetArrayParamsFromObject); #undef DECL_NPY_API - return api; - } - - bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); } - - PyObject *(*PyArray_DescrFromType_)(int); - PyObject *(*PyArray_NewFromDescr_) - (PyTypeObject *, PyObject *, int, Py_intptr_t *, - Py_intptr_t *, void *, int, PyObject *); - PyObject *(*PyArray_DescrNewFromType_)(int); - PyObject *(*PyArray_NewCopy_)(PyObject *, int); - PyTypeObject *PyArray_Type_; - PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); - int (*PyArray_DescrConverter_) (PyObject *, PyObject **); - bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); - int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, - Py_ssize_t *, PyObject **, PyObject *); - }; + return api; + } +}; +} - PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_) +class array : public buffer { +public: + PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) enum { - c_style = API::NPY_C_CONTIGUOUS_, - f_style = API::NPY_F_CONTIGUOUS_, - forcecast = API::NPY_ARRAY_FORCECAST_ + c_style = detail::npy_api::NPY_C_CONTIGUOUS_, + f_style = detail::npy_api::NPY_F_CONTIGUOUS_, + forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; template array(size_t size, const Type *ptr) { - API& api = lookup_api(); + auto& api = detail::npy_api::get(); PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); Py_intptr_t shape = (Py_intptr_t) size; object tmp = object(api.PyArray_NewFromDescr_( @@ -121,7 +128,7 @@ public: } array(const buffer_info &info) { - auto& api = lookup_api(); + auto& api = detail::npy_api::get(); // _dtype_from_pep3118 returns dtypes with padding fields in, so we need to strip them auto numpy_internal = module::import("numpy.core._internal"); @@ -139,11 +146,6 @@ public: } protected: - static API &lookup_api() { - static API api = API::lookup(); - return api; - } - template friend struct detail::npy_format_descriptor; static object strip_padding_fields(object dtype) { @@ -183,7 +185,7 @@ protected: args["itemsize"] = dtype.attr("itemsize").cast(); PyObject *descr = nullptr; - if (!lookup_api().PyArray_DescrConverter_(args.release().ptr(), &descr) || !descr) + if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &descr) || !descr) pybind11_fail("NumPy: failed to create structured dtype"); return object(descr, false); } @@ -198,10 +200,10 @@ public: static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) return nullptr; - API &api = lookup_api(); + auto& api = detail::npy_api::get(); PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, - API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); Py_DECREF(ptr); @@ -246,12 +248,12 @@ struct is_pod_struct { template struct npy_format_descriptor::value>::type> { private: constexpr static const int values[8] = { - array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_, array::API::NPY_USHORT_, - array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; + npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, + npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ }; public: enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; static object dtype() { - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(value)) + if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) return object(ptr, true); pybind11_fail("Unsupported buffer format!"); } @@ -264,9 +266,9 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - enum { value = array::API::NumPyName }; \ + enum { value = npy_api::NumPyName }; \ static object dtype() { \ - if (auto ptr = array::lookup_api().PyArray_DescrFromType_(value)) \ + if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ return object(ptr, true); \ pybind11_fail("Unsupported buffer format!"); \ } \ @@ -281,7 +283,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #define DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _(); } \ static object dtype() { \ - auto& api = array::lookup_api(); \ + auto& api = npy_api::get(); \ PyObject *descr = nullptr; \ PYBIND11_DESCR fmt = _("S") + _(); \ pybind11::str py_fmt(fmt.text()); \ @@ -319,7 +321,7 @@ struct npy_format_descriptor::value> } static void register_dtype(std::initializer_list fields) { - auto& api = array::lookup_api(); + auto& api = npy_api::get(); auto args = dict(); list names { }, offsets { }, formats { }; for (auto field : fields) { -- cgit v1.2.3 From 01f74095504211d2587ea3f48b2ffbe51b5ad724 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 23 Jul 2016 21:55:37 +0100 Subject: Initial implementation of py::dtype --- include/pybind11/numpy.h | 178 +++++++++++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 74 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 77f7e6f..3b52fa3 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -52,7 +52,12 @@ struct npy_api { return api; } - bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); } + bool PyArray_Check_(PyObject *obj) const { + return (bool) PyObject_TypeCheck(obj, PyArray_Type_); + } + bool PyArrayDescr_Check_(PyObject *obj) const { + return (bool) PyObject_TypeCheck(obj, PyArrayDescr_Type_); + } PyObject *(*PyArray_DescrFromType_)(int); PyObject *(*PyArray_NewFromDescr_) @@ -61,6 +66,7 @@ struct npy_api { PyObject *(*PyArray_DescrNewFromType_)(int); PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; + PyTypeObject *PyArrayDescr_Type_; PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); int (*PyArray_DescrConverter_) (PyObject *, PyObject **); bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); @@ -69,6 +75,7 @@ struct npy_api { private: enum functions { API_PyArray_Type = 2, + API_PyArrayDescr_Type = 3, API_PyArray_DescrFromType = 45, API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, @@ -90,6 +97,7 @@ private: npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_Type); + DECL_NPY_API(PyArrayDescr_Type); DECL_NPY_API(PyArray_DescrFromType); DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); @@ -104,56 +112,55 @@ private: }; } -class array : public buffer { +class dtype : public object { public: - PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) + PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); - enum { - c_style = detail::npy_api::NPY_C_CONTIGUOUS_, - f_style = detail::npy_api::NPY_F_CONTIGUOUS_, - forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ - }; + dtype(const buffer_info &info) { + dtype descr(_dtype_from_pep3118()(pybind11::str(info.format))); + m_ptr = descr.strip_padding().release().ptr(); + } - template array(size_t size, const Type *ptr) { - auto& api = detail::npy_api::get(); - PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); - Py_intptr_t shape = (Py_intptr_t) size; - object tmp = object(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - if (ptr) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - m_ptr = tmp.release().ptr(); + dtype(std::string format) { + m_ptr = from_args(pybind11::str(format)).release().ptr(); } - array(const buffer_info &info) { - auto& api = detail::npy_api::get(); + static dtype from_args(object args) { + // This is essentially the same as calling np.dtype() constructor in Python + PyObject *ptr = nullptr; + if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr) + pybind11_fail("NumPy: failed to create structured dtype"); + return object(ptr, false); + } - // _dtype_from_pep3118 returns dtypes with padding fields in, so we need to strip them - auto numpy_internal = module::import("numpy.core._internal"); - auto dtype_from_fmt = (object) numpy_internal.attr("_dtype_from_pep3118"); - auto dtype = strip_padding_fields(dtype_from_fmt(pybind11::str(info.format))); + template static dtype of() { + return detail::npy_format_descriptor::dtype(); + } - object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, dtype.release().ptr(), (int) info.ndim, (Py_intptr_t *) &info.shape[0], - (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - if (info.ptr) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - m_ptr = tmp.release().ptr(); + size_t itemsize() const { + return (size_t) attr("itemsize").cast(); } -protected: - template friend struct detail::npy_format_descriptor; + bool has_fields() const { + return attr("fields").cast().ptr() != Py_None; + } + + std::string kind() const { + return (std::string) attr("kind").cast(); + } + +private: + static object& _dtype_from_pep3118() { + static object obj = module::import("numpy.core._internal").attr("_dtype_from_pep3118"); + return obj; + } - static object strip_padding_fields(object dtype) { + dtype strip_padding() { // Recursively strip all void fields with empty names that are generated for // padding fields (as of NumPy v1.11). - auto fields = dtype.attr("fields").cast(); + auto fields = attr("fields").cast(); if (fields.ptr() == Py_None) - return dtype; + return *this; struct field_descr { pybind11::str name; object format; int_ offset; }; std::vector field_descriptors; @@ -162,11 +169,11 @@ protected: for (auto field : items()) { auto spec = object(field, true).cast(); auto name = spec[0].cast(); - auto format = spec[1].cast()[0].cast(); + auto format = spec[1].cast()[0].cast(); auto offset = spec[1].cast()[1].cast(); - if (!len(name) && (std::string) dtype.attr("kind").cast() == "V") + if (!len(name) && format.kind() == "V") continue; - field_descriptors.push_back({name, strip_padding_fields(format), offset}); + field_descriptors.push_back({name, format.strip_padding(), offset}); } std::sort(field_descriptors.begin(), field_descriptors.end(), @@ -176,19 +183,57 @@ protected: list names, formats, offsets; for (auto& descr : field_descriptors) { - names.append(descr.name); - formats.append(descr.format); - offsets.append(descr.offset); + names.append(descr.name); formats.append(descr.format); offsets.append(descr.offset); } auto args = dict(); args["names"] = names; args["formats"] = formats; args["offsets"] = offsets; - args["itemsize"] = dtype.attr("itemsize").cast(); + args["itemsize"] = (int_) itemsize(); + return dtype::from_args(args); + } +}; - PyObject *descr = nullptr; - if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &descr) || !descr) - pybind11_fail("NumPy: failed to create structured dtype"); - return object(descr, false); +class array : public buffer { +public: + PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) + + enum { + c_style = detail::npy_api::NPY_C_CONTIGUOUS_, + f_style = detail::npy_api::NPY_F_CONTIGUOUS_, + forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ + }; + + template array(size_t size, const Type *ptr) { + auto& api = detail::npy_api::get(); + auto descr = pybind11::dtype::of().release().ptr(); + Py_intptr_t shape = (Py_intptr_t) size; + object tmp = object(api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); + if (!tmp) + pybind11_fail("NumPy: unable to create array!"); + if (ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + m_ptr = tmp.release().ptr(); + } + + array(const buffer_info &info) { + auto& api = detail::npy_api::get(); + auto descr = pybind11::dtype(info).release().ptr(); + object tmp(api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0], + (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); + if (!tmp) + pybind11_fail("NumPy: unable to create array!"); + if (info.ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + m_ptr = tmp.release().ptr(); } + + pybind11::dtype dtype() { + return attr("dtype").cast(); + } + +protected: + template friend struct detail::npy_format_descriptor; }; template class array_t : public array { @@ -201,8 +246,7 @@ public: if (ptr == nullptr) return nullptr; auto& api = detail::npy_api::get(); - PyObject *descr = detail::npy_format_descriptor::dtype().release().ptr(); - PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, + PyObject *result = api.PyArray_FromAny_(ptr, pybind11::dtype::of().release().ptr(), 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); @@ -223,11 +267,6 @@ template struct format_descriptor> { static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } }; -template -object dtype_of() { - return detail::npy_format_descriptor::dtype(); -} - NAMESPACE_BEGIN(detail) template struct is_std_array : std::false_type { }; template struct is_std_array> : std::true_type { }; @@ -252,7 +291,7 @@ private: npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ }; public: enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; - static object dtype() { + static pybind11::dtype dtype() { if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) return object(ptr, true); pybind11_fail("Unsupported buffer format!"); @@ -267,7 +306,7 @@ template constexpr const int npy_format_descriptor< #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ enum { value = npy_api::NumPyName }; \ - static object dtype() { \ + static pybind11::dtype dtype() { \ if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ return object(ptr, true); \ pybind11_fail("Unsupported buffer format!"); \ @@ -282,14 +321,9 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #define DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _(); } \ - static object dtype() { \ - auto& api = npy_api::get(); \ - PyObject *descr = nullptr; \ + static pybind11::dtype dtype() { \ PYBIND11_DESCR fmt = _("S") + _(); \ - pybind11::str py_fmt(fmt.text()); \ - if (!api.PyArray_DescrConverter_(py_fmt.release().ptr(), &descr) || !descr) \ - pybind11_fail("NumPy: failed to create string dtype"); \ - return object(descr, false); \ + return pybind11::dtype::from_args(pybind11::str(fmt.text())); \ } \ static const char *format() { PYBIND11_DESCR s = _() + _("s"); return s.text(); } template struct npy_format_descriptor { DECL_CHAR_FMT }; @@ -301,14 +335,14 @@ struct field_descriptor { size_t offset; size_t size; const char *format; - object descr; + dtype descr; }; template struct npy_format_descriptor::value>::type> { static PYBIND11_DESCR name() { return _("struct"); } - static object dtype() { + static pybind11::dtype dtype() { if (!dtype_()) pybind11_fail("NumPy: unsupported buffer format!"); return object(dtype_(), true); @@ -321,7 +355,6 @@ struct npy_format_descriptor::value> } static void register_dtype(std::initializer_list fields) { - auto& api = npy_api::get(); auto args = dict(); list names { }, offsets { }, formats { }; for (auto field : fields) { @@ -333,10 +366,7 @@ struct npy_format_descriptor::value> } args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; args["itemsize"] = int_(sizeof(T)); - // This is essentially the same as calling np.dtype() constructor in Python and passing - // it a dict of the form {'names': ..., 'formats': ..., 'offsets': ...}. - if (!api.PyArray_DescrConverter_(args.release().ptr(), &dtype_()) || !dtype_()) - pybind11_fail("NumPy: failed to create structured dtype"); + dtype_() = pybind11::dtype::from_args(args).release().ptr(); // There is an existing bug in NumPy (as of v1.11): trailing bytes are // not encoded explicitly into the format string. This will supposedly @@ -366,9 +396,9 @@ struct npy_format_descriptor::value> format_() = oss.str(); // Sanity check: verify that NumPy properly parses our buffer format string + auto& api = npy_api::get(); auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); - auto fixed_dtype = array::strip_padding_fields(object(dtype_(), true)); - if (!api.PyArray_EquivTypes_(dtype_(), fixed_dtype.ptr())) + if (!api.PyArray_EquivTypes_(dtype_(), arr.dtype().ptr())) pybind11_fail("NumPy: invalid buffer descriptor!"); } -- cgit v1.2.3 From fc5620afa68a802b5953c2ef05b74611a908a054 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 17:34:53 +0100 Subject: Fix a segfault where func object wasn't released --- include/pybind11/numpy.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3b52fa3..3956c34 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -150,9 +150,10 @@ public: } private: - static object& _dtype_from_pep3118() { - static object obj = module::import("numpy.core._internal").attr("_dtype_from_pep3118"); - return obj; + static object _dtype_from_pep3118() { + static PyObject *obj = module::import("numpy.core._internal") + .attr("_dtype_from_pep3118").cast().release().ptr(); + return object(obj, true); } dtype strip_padding() { -- cgit v1.2.3 From d77bc8c343cb62c8a4b13fa50b523500099d59c6 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 17:51:35 +0100 Subject: Add dtype(names, offsets, formats, itemsize) ctor --- include/pybind11/numpy.h | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3956c34..d048875 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -125,6 +125,15 @@ public: m_ptr = from_args(pybind11::str(format)).release().ptr(); } + dtype(list names, list formats, list offsets, size_t itemsize) { + dict args; + args["names"] = names; + args["formats"] = formats; + args["offsets"] = offsets; + args["itemsize"] = int_(itemsize); + m_ptr = from_args(args).release().ptr(); + } + static dtype from_args(object args) { // This is essentially the same as calling np.dtype() constructor in Python PyObject *ptr = nullptr; @@ -184,12 +193,11 @@ private: list names, formats, offsets; for (auto& descr : field_descriptors) { - names.append(descr.name); formats.append(descr.format); offsets.append(descr.offset); + names.append(descr.name); + formats.append(descr.format); + offsets.append(descr.offset); } - auto args = dict(); - args["names"] = names; args["formats"] = formats; args["offsets"] = offsets; - args["itemsize"] = (int_) itemsize(); - return dtype::from_args(args); + return dtype(names, formats, offsets, itemsize()); } }; @@ -324,7 +332,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); static PYBIND11_DESCR name() { return _("S") + _(); } \ static pybind11::dtype dtype() { \ PYBIND11_DESCR fmt = _("S") + _(); \ - return pybind11::dtype::from_args(pybind11::str(fmt.text())); \ + return pybind11::dtype(fmt.text()); \ } \ static const char *format() { PYBIND11_DESCR s = _() + _("s"); return s.text(); } template struct npy_format_descriptor { DECL_CHAR_FMT }; @@ -356,18 +364,15 @@ struct npy_format_descriptor::value> } static void register_dtype(std::initializer_list fields) { - auto args = dict(); - list names { }, offsets { }, formats { }; + list names, formats, offsets; for (auto field : fields) { if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); names.append(str(field.name)); - offsets.append(int_(field.offset)); formats.append(field.descr); + offsets.append(int_(field.offset)); } - args["names"] = names; args["offsets"] = offsets; args["formats"] = formats; - args["itemsize"] = int_(sizeof(T)); - dtype_() = pybind11::dtype::from_args(args).release().ptr(); + dtype_() = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); // There is an existing bug in NumPy (as of v1.11): trailing bytes are // not encoded explicitly into the format string. This will supposedly -- cgit v1.2.3 From 6bb0ee1186de21b59455b87e2e374e11e75cc48d Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 18:35:14 +0100 Subject: Add all possible ctors for py::array --- include/pybind11/numpy.h | 62 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 17 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d048875..0d4aeaf 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -211,12 +211,16 @@ public: forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; - template array(size_t size, const Type *ptr) { + array(const pybind11::dtype& dt, const std::vector& shape, + void *ptr, const std::vector& strides) { auto& api = detail::npy_api::get(); - auto descr = pybind11::dtype::of().release().ptr(); - Py_intptr_t shape = (Py_intptr_t) size; - object tmp = object(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); + auto ndim = shape.size(); + if (shape.size() != strides.size()) + pybind11_fail("NumPy: shape ndim doesn't match strides ndim"); + auto descr = dt; + object tmp(api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), + (Py_intptr_t *) strides.data(), ptr, 0, nullptr), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); if (ptr) @@ -224,18 +228,30 @@ public: m_ptr = tmp.release().ptr(); } - array(const buffer_info &info) { - auto& api = detail::npy_api::get(); - auto descr = pybind11::dtype(info).release().ptr(); - object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0], - (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - if (info.ptr) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - m_ptr = tmp.release().ptr(); - } + array(const pybind11::dtype& dt, const std::vector& shape, void *ptr) + : array(dt, shape, ptr, default_strides(shape, dt.itemsize())) + { } + + array(const pybind11::dtype& dt, size_t size, void *ptr) + : array(dt, std::vector { size }, ptr) + { } + + template array(const std::vector& shape, + T* ptr, const std::vector& strides) + : array(pybind11::dtype::of(), shape, (void *) ptr, strides) + { } + + template array(const std::vector& shape, T* ptr) + : array(shape, ptr, default_strides(shape, sizeof(T))) + { } + + template array(size_t size, T* ptr) + : array(std::vector { size }, ptr) + { } + + array(const buffer_info &info) + : array(pybind11::dtype(info), info.shape, info.ptr, info.strides) + { } pybind11::dtype dtype() { return attr("dtype").cast(); @@ -243,6 +259,18 @@ public: protected: template friend struct detail::npy_format_descriptor; + + static std::vector default_strides(const std::vector& shape, size_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim); + if (ndim) { + std::fill(strides.begin(), strides.end(), itemsize); + for (size_t i = 0; i < ndim - 1; i++) + for (size_t j = 0; j < ndim - 1 - i; j++) + strides[j] *= shape[ndim - 1 - i]; + } + return strides; + } }; template class array_t : public array { -- cgit v1.2.3 From 6636ae9d4e6c9de5bc33a6f0fd40b72f90a5a888 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 24 Jul 2016 18:54:53 +0100 Subject: Also add the new ctors to py::array_t --- include/pybind11/numpy.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0d4aeaf..d7b697f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -276,8 +276,21 @@ protected: template class array_t : public array { public: PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); + array_t() : array() { } - array_t(const buffer_info& info) : array(info) {} + + array_t(const buffer_info& info) : array(info) { } + + array_t(const std::vector& shape, + T* ptr, const std::vector& strides) + : array(shape, ptr, strides) { } + + array_t(const std::vector& shape, T* ptr) + : array(shape, ptr) { } + + array_t(size_t size, T* ptr) + : array(size, ptr) { } + static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) -- cgit v1.2.3 From c6257f864112791e68241c6676b492a0fdd6e00b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 25 Jul 2016 00:46:39 +0100 Subject: Allow nullptr in array ctors wherever possible --- include/pybind11/numpy.h | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d7b697f..b9ee69e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -212,7 +212,7 @@ public: }; array(const pybind11::dtype& dt, const std::vector& shape, - void *ptr, const std::vector& strides) { + const std::vector& strides, void *ptr = nullptr) { auto& api = detail::npy_api::get(); auto ndim = shape.size(); if (shape.size() != strides.size()) @@ -228,30 +228,24 @@ public: m_ptr = tmp.release().ptr(); } - array(const pybind11::dtype& dt, const std::vector& shape, void *ptr) - : array(dt, shape, ptr, default_strides(shape, dt.itemsize())) - { } + array(const pybind11::dtype& dt, const std::vector& shape, void *ptr = nullptr) + : array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { } - array(const pybind11::dtype& dt, size_t size, void *ptr) - : array(dt, std::vector { size }, ptr) - { } + array(const pybind11::dtype& dt, size_t size, void *ptr = nullptr) + : array(dt, std::vector { size }, ptr) { } template array(const std::vector& shape, - T* ptr, const std::vector& strides) - : array(pybind11::dtype::of(), shape, (void *) ptr, strides) - { } + const std::vector& strides, T* ptr) + : array(pybind11::dtype::of(), shape, strides, (void *) ptr) { } template array(const std::vector& shape, T* ptr) - : array(shape, ptr, default_strides(shape, sizeof(T))) - { } + : array(shape, default_strides(shape, sizeof(T)), ptr) { } template array(size_t size, T* ptr) - : array(std::vector { size }, ptr) - { } + : array(std::vector { size }, ptr) { } array(const buffer_info &info) - : array(pybind11::dtype(info), info.shape, info.ptr, info.strides) - { } + : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } pybind11::dtype dtype() { return attr("dtype").cast(); @@ -281,17 +275,17 @@ public: array_t(const buffer_info& info) : array(info) { } - array_t(const std::vector& shape, - T* ptr, const std::vector& strides) - : array(shape, ptr, strides) { } + array_t(const std::vector& shape, const std::vector& strides, T* ptr = nullptr) + : array(shape, strides, ptr) { } - array_t(const std::vector& shape, T* ptr) + array_t(const std::vector& shape, T* ptr = nullptr) : array(shape, ptr) { } - array_t(size_t size, T* ptr) + array_t(size_t size, T* ptr = nullptr) : array(size, ptr) { } static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } + static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) return nullptr; -- cgit v1.2.3 From ad5ca6d4e64c8d8521567fe9ac982fafc793d021 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 25 Jul 2016 00:58:17 +0100 Subject: Added dtype from const char pointer ctor --- include/pybind11/numpy.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b9ee69e..59530d4 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -125,6 +125,8 @@ public: m_ptr = from_args(pybind11::str(format)).release().ptr(); } + dtype(const char *format) : dtype(std::string(format)) { } + dtype(list names, list formats, list offsets, size_t itemsize) { dict args; args["names"] = names; -- cgit v1.2.3 From 61e3b0bd1580bde20e64514f534eb662a2ae1957 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 13 Aug 2016 12:42:02 +0100 Subject: Use builtin str type for recarray field names --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 59530d4..c71d9bb 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -117,7 +117,7 @@ public: PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); dtype(const buffer_info &info) { - dtype descr(_dtype_from_pep3118()(pybind11::str(info.format))); + dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format))); m_ptr = descr.strip_padding().release().ptr(); } @@ -174,7 +174,7 @@ private: if (fields.ptr() == Py_None) return *this; - struct field_descr { pybind11::str name; object format; int_ offset; }; + struct field_descr { PYBIND11_STR_TYPE name; object format; int_ offset; }; std::vector field_descriptors; auto items = fields.attr("items").cast(); @@ -185,7 +185,7 @@ private: auto offset = spec[1].cast()[1].cast(); if (!len(name) && format.kind() == "V") continue; - field_descriptors.push_back({name, format.strip_padding(), offset}); + field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(), offset}); } std::sort(field_descriptors.begin(), field_descriptors.end(), @@ -405,7 +405,7 @@ struct npy_format_descriptor::value> for (auto field : fields) { if (!field.descr) pybind11_fail("NumPy: unsupported field dtype"); - names.append(str(field.name)); + names.append(PYBIND11_STR_TYPE(field.name)); formats.append(field.descr); offsets.append(int_(field.offset)); } -- cgit v1.2.3 From 03fb488579b271a26e0865ffd4820ef4f9d7caf1 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 14 Aug 2016 13:45:49 +0100 Subject: format_descriptor::format() now yields std::string This is required since format descriptors for string types that were using PYBIND11_DESCR were causing problems on C++14 on Linux. Although this is technically a breaking change, it shouldn't cause problems since the only use of format strings is passing them to buffer_info constructor which expects std::string. Note: for non-structured types, the const char * value is still accessible via ::value for compatibility purpose. --- include/pybind11/numpy.h | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index c71d9bb..8653fc1 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #if defined(_MSC_VER) @@ -303,14 +304,14 @@ public: template struct format_descriptor::value>::type> { - static const char *format() { return detail::npy_format_descriptor::format(); } + static std::string format() { return detail::npy_format_descriptor::format(); } }; template struct format_descriptor { - static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } + static std::string format() { return std::to_string(N) + "s"; } }; template struct format_descriptor> { - static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } + static std::string format() { return std::to_string(N) + "s"; } }; NAMESPACE_BEGIN(detail) @@ -367,11 +368,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #define DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _(); } \ - static pybind11::dtype dtype() { \ - PYBIND11_DESCR fmt = _("S") + _(); \ - return pybind11::dtype(fmt.text()); \ - } \ - static const char *format() { PYBIND11_DESCR s = _() + _("s"); return s.text(); } + static pybind11::dtype dtype() { return std::string("S") + std::to_string(N); } template struct npy_format_descriptor { DECL_CHAR_FMT }; template struct npy_format_descriptor> { DECL_CHAR_FMT }; #undef DECL_CHAR_FMT @@ -380,7 +377,7 @@ struct field_descriptor { const char *name; size_t offset; size_t size; - const char *format; + std::string format; dtype descr; }; @@ -389,15 +386,15 @@ struct npy_format_descriptor::value> static PYBIND11_DESCR name() { return _("struct"); } static pybind11::dtype dtype() { - if (!dtype_()) + if (!dtype_ptr) pybind11_fail("NumPy: unsupported buffer format!"); - return object(dtype_(), true); + return object(dtype_ptr, true); } - static const char* format() { - if (!dtype_()) + static std::string format() { + if (!dtype_ptr) pybind11_fail("NumPy: unsupported buffer format!"); - return format_().c_str(); + return format_str; } static void register_dtype(std::initializer_list fields) { @@ -409,7 +406,7 @@ struct npy_format_descriptor::value> formats.append(field.descr); offsets.append(int_(field.offset)); } - dtype_() = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); + dtype_ptr = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); // There is an existing bug in NumPy (as of v1.11): trailing bytes are // not encoded explicitly into the format string. This will supposedly @@ -436,20 +433,25 @@ struct npy_format_descriptor::value> if (sizeof(T) > offset) oss << (sizeof(T) - offset) << 'x'; oss << '}'; - format_() = oss.str(); + format_str = oss.str(); // Sanity check: verify that NumPy properly parses our buffer format string auto& api = npy_api::get(); - auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); - if (!api.PyArray_EquivTypes_(dtype_(), arr.dtype().ptr())) + auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1)); + if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) pybind11_fail("NumPy: invalid buffer descriptor!"); } private: - static inline PyObject*& dtype_() { static PyObject *ptr = nullptr; return ptr; } - static inline std::string& format_() { static std::string s; return s; } + static std::string format_str; + static PyObject* dtype_ptr; }; +template +std::string npy_format_descriptor::value>::type>::format_str; +template +PyObject* npy_format_descriptor::value>::type>::dtype_ptr = nullptr; + // Extract name, offset and format descriptor for a struct field #define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ ::pybind11::detail::field_descriptor { \ -- cgit v1.2.3 From edbd4cb0a7f8a67280be27c7f7dcdafbed60a010 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 15 Aug 2016 01:23:59 +0100 Subject: Decay const qualifiers in is_pod_struct<> --- include/pybind11/numpy.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8653fc1..e2b5a16 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -29,6 +29,8 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; template struct is_pod_struct; +template using decay_cv_ref = + typename std::remove_cv::type>::type; struct npy_api { enum constants { @@ -321,14 +323,15 @@ template struct is_std_array> : std::tru template struct is_pod_struct { enum { value = std::is_pod::value && // offsetof only works correctly for POD types + !std::is_reference::value && !std::is_array::value && !is_std_array::value && !std::is_integral::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same::value && - !std::is_same>::value && - !std::is_same>::value }; + !std::is_same::type, float>::value && + !std::is_same::type, double>::value && + !std::is_same::type, bool>::value && + !std::is_same::type, std::complex>::value && + !std::is_same::type, std::complex>::value }; }; template struct npy_format_descriptor::value>::type> { -- cgit v1.2.3 From 67b3daeea4c621849ba4636d1d9865c14d089629 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 15 Aug 2016 01:24:28 +0100 Subject: Always decay type param of npy_format_descriptor --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e2b5a16..57961c5 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -29,8 +29,6 @@ NAMESPACE_BEGIN(pybind11) namespace detail { template struct npy_format_descriptor { }; template struct is_pod_struct; -template using decay_cv_ref = - typename std::remove_cv::type>::type; struct npy_api { enum constants { @@ -148,7 +146,7 @@ public: } template static dtype of() { - return detail::npy_format_descriptor::dtype(); + return detail::npy_format_descriptor::type>::dtype(); } size_t itemsize() const { @@ -306,7 +304,9 @@ public: template struct format_descriptor::value>::type> { - static std::string format() { return detail::npy_format_descriptor::format(); } + static std::string format() { + return detail::npy_format_descriptor::type>::format(); + } }; template struct format_descriptor { -- cgit v1.2.3 From 1c8828fe8f408cf864c1182dd3e0f103220540de Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Thu, 25 Aug 2016 00:33:02 +0100 Subject: Fix int_ shadowing problem in detail namespace If operators.h is included, int_ function in the `detail` namespace will shadow pybind11::int_ type, so the fully qualified name has to be used. --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 57961c5..79c4cf3 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -407,7 +407,7 @@ struct npy_format_descriptor::value> pybind11_fail("NumPy: unsupported field dtype"); names.append(PYBIND11_STR_TYPE(field.name)); formats.append(field.descr); - offsets.append(int_(field.offset)); + offsets.append(pybind11::int_(field.offset)); } dtype_ptr = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); -- cgit v1.2.3 From 9a777a263dba76bbbe1cff173142d3e30b1363b3 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Thu, 25 Aug 2016 02:16:47 +0200 Subject: numpy.h: fix test suite issues on the Intel Compiler --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 79c4cf3..3fe426a 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -133,7 +133,7 @@ public: args["names"] = names; args["formats"] = formats; args["offsets"] = offsets; - args["itemsize"] = int_(itemsize); + args["itemsize"] = pybind11::int_(itemsize); m_ptr = from_args(args).release().ptr(); } @@ -150,7 +150,7 @@ public: } size_t itemsize() const { - return (size_t) attr("itemsize").cast(); + return attr("itemsize").cast(); } bool has_fields() const { @@ -175,7 +175,7 @@ private: if (fields.ptr() == Py_None) return *this; - struct field_descr { PYBIND11_STR_TYPE name; object format; int_ offset; }; + struct field_descr { PYBIND11_STR_TYPE name; object format; pybind11::int_ offset; }; std::vector field_descriptors; auto items = fields.attr("items").cast(); @@ -183,7 +183,7 @@ private: auto spec = object(field, true).cast(); auto name = spec[0].cast(); auto format = spec[1].cast()[0].cast(); - auto offset = spec[1].cast()[1].cast(); + auto offset = spec[1].cast()[1].cast(); if (!len(name) && format.kind() == "V") continue; field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(), offset}); -- cgit v1.2.3 From d8b11b8708df720e903f423e3a3e4370554a1e45 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Thu, 25 Aug 2016 21:52:52 +0100 Subject: Fix dtype::strip_padding() on Intel compiler --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3fe426a..c06fb26 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -191,7 +191,7 @@ private: std::sort(field_descriptors.begin(), field_descriptors.end(), [](const field_descr& a, const field_descr& b) { - return (int) a.offset < (int) b.offset; + return a.offset.cast() < b.offset.cast(); }); list names, formats, offsets; -- cgit v1.2.3 From a3906778eb86bc1885bffd86ef61838770a2624b Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sat, 27 Aug 2016 13:09:02 +0200 Subject: minor: renamed argument in array constructor --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index c06fb26..51c68ad 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -234,8 +234,8 @@ public: array(const pybind11::dtype& dt, const std::vector& shape, void *ptr = nullptr) : array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { } - array(const pybind11::dtype& dt, size_t size, void *ptr = nullptr) - : array(dt, std::vector { size }, ptr) { } + array(const pybind11::dtype& dt, size_t count, void *ptr = nullptr) + : array(dt, std::vector { count }, ptr) { } template array(const std::vector& shape, const std::vector& strides, T* ptr) -- cgit v1.2.3 From 91b3d681ad96951bd51a719897b89487c3e99d37 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 29 Aug 2016 02:41:05 +0100 Subject: Expose some dtype/array attributes via NumPy C API --- include/pybind11/numpy.h | 151 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 131 insertions(+), 20 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 51c68ad..6e0785e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -19,6 +19,7 @@ #include #include #include +#include #if defined(_MSC_VER) #pragma warning(push) @@ -30,12 +31,41 @@ namespace detail { template struct npy_format_descriptor { }; template struct is_pod_struct; +struct PyArrayDescr_Proxy { + PyObject_HEAD + PyObject *typeobj; + char kind; + char type; + char byteorder; + char flags; + int type_num; + int elsize; + int alignment; + char *subarray; + PyObject *fields; + PyObject *names; +}; + +struct PyArray_Proxy { + PyObject_HEAD + char *data; + int nd; + ssize_t *dimensions; + ssize_t *strides; + PyObject *base; + PyObject *descr; + int flags; +}; + struct npy_api { enum constants { NPY_C_CONTIGUOUS_ = 0x0001, NPY_F_CONTIGUOUS_ = 0x0002, + NPY_ARRAY_OWNDATA_ = 0x0004, NPY_ARRAY_FORCECAST_ = 0x0010, NPY_ENSURE_ARRAY_ = 0x0040, + NPY_ARRAY_ALIGNED_ = 0x0100, + NPY_ARRAY_WRITEABLE_ = 0x0400, NPY_BOOL_ = 0, NPY_BYTE_, NPY_UBYTE_, NPY_SHORT_, NPY_USHORT_, @@ -113,6 +143,11 @@ private: }; } +#define PyArray_GET_(ptr, attr) (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->attr) +#define PyArrayDescr_GET_(ptr, attr) (reinterpret_cast<::pybind11::detail::PyArrayDescr_Proxy*>(ptr)->attr) +#define PyArray_CHKFLAGS_(ptr, flag) \ + (flag == (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->flags & flag)) + class dtype : public object { public: PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); @@ -150,15 +185,15 @@ public: } size_t itemsize() const { - return attr("itemsize").cast(); + return (size_t) PyArrayDescr_GET_(m_ptr, elsize); } bool has_fields() const { - return attr("fields").cast().ptr() != Py_None; + return PyArrayDescr_GET_(m_ptr, names) != nullptr; } - std::string kind() const { - return (std::string) attr("kind").cast(); + char kind() const { + return PyArrayDescr_GET_(m_ptr, kind); } private: @@ -171,20 +206,20 @@ private: dtype strip_padding() { // Recursively strip all void fields with empty names that are generated for // padding fields (as of NumPy v1.11). - auto fields = attr("fields").cast(); - if (fields.ptr() == Py_None) + if (!has_fields()) return *this; struct field_descr { PYBIND11_STR_TYPE name; object format; pybind11::int_ offset; }; std::vector field_descriptors; + auto fields = attr("fields").cast(); auto items = fields.attr("items").cast(); for (auto field : items()) { auto spec = object(field, true).cast(); auto name = spec[0].cast(); auto format = spec[1].cast()[0].cast(); auto offset = spec[1].cast()[1].cast(); - if (!len(name) && format.kind() == "V") + if (!len(name) && format.kind() == 'V') continue; field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(), offset}); } @@ -244,14 +279,83 @@ public: template array(const std::vector& shape, T* ptr) : array(shape, default_strides(shape, sizeof(T)), ptr) { } - template array(size_t size, T* ptr) - : array(std::vector { size }, ptr) { } + template array(size_t count, T* ptr) + : array(std::vector { count }, ptr) { } array(const buffer_info &info) : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } - pybind11::dtype dtype() { - return attr("dtype").cast(); + /// Array descriptor (dtype) + pybind11::dtype dtype() const { + return object(PyArray_GET_(m_ptr, descr), true); + } + + /// Total number of elements + size_t size() const { + return std::accumulate(shape(), shape() + ndim(), (size_t) 1, std::multiplies()); + } + + /// Byte size of a single element + size_t itemsize() const { + return (size_t) PyArrayDescr_GET_(PyArray_GET_(m_ptr, descr), elsize); + } + + /// Total number of bytes + size_t nbytes() const { + return size() * itemsize(); + } + + /// Number of dimensions + size_t ndim() const { + return (size_t) PyArray_GET_(m_ptr, nd); + } + + /// Dimensions of the array + const size_t* shape() const { + static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); + return reinterpret_cast(PyArray_GET_(m_ptr, dimensions)); + } + + /// Dimension along a given axis + size_t shape(size_t dim) const { + if (dim >= ndim()) + pybind11_fail("NumPy: attempted to index shape beyond ndim"); + return shape()[dim]; + } + + /// Strides of the array + const size_t* strides() const { + static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); + return reinterpret_cast(PyArray_GET_(m_ptr, strides)); + } + + /// Stride along a given axis + size_t strides(size_t dim) const { + if (dim >= ndim()) + pybind11_fail("NumPy: attempted to index strides beyond ndim"); + return strides()[dim]; + } + + /// If set, the array is writeable (otherwise the buffer is read-only) + bool writeable() const { + return PyArray_CHKFLAGS_(m_ptr, detail::npy_api::NPY_ARRAY_WRITEABLE_); + } + + /// If set, the array owns the data (will be freed when the array is deleted) + bool owndata() const { + return PyArray_CHKFLAGS_(m_ptr, detail::npy_api::NPY_ARRAY_OWNDATA_); + } + + /// Direct pointer to contained buffer + const void* data() const { + return reinterpret_cast(PyArray_GET_(m_ptr, data)); + } + + /// Direct mutable pointer to contained buffer (checks writeable flag) + void* mutable_data() { + if (!writeable()) + pybind11_fail("NumPy: cannot get mutable data of a read-only array"); + return reinterpret_cast(PyArray_GET_(m_ptr, data)); } protected: @@ -284,8 +388,18 @@ public: array_t(const std::vector& shape, T* ptr = nullptr) : array(shape, ptr) { } - array_t(size_t size, T* ptr = nullptr) - : array(size, ptr) { } + array_t(size_t count, T* ptr = nullptr) + : array(count, ptr) { } + + const T* data() const { + return reinterpret_cast(PyArray_GET_(m_ptr, data)); + } + + T* mutable_data() { + if (!writeable()) + pybind11_fail("NumPy: cannot get mutable data of a read-only array"); + return reinterpret_cast(PyArray_GET_(m_ptr, data)); + } static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } @@ -678,16 +792,13 @@ struct vectorize_helper { if (size == 1) return cast(f(*((Args *) buffers[Index].ptr)...)); - array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::format(), - ndim, shape, strides)); - - buffer_info buf = result.request(); - Return *output = (Return *) buf.ptr; + array_t result(shape, strides); + auto buf = result.request(); + auto output = (Return *) buf.ptr; if (trivial_broadcast) { /* Call the function */ - for (size_t i=0; i Date: Thu, 8 Sep 2016 21:48:14 +0100 Subject: array: add direct data access and indexing methods --- include/pybind11/numpy.h | 135 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 102 insertions(+), 33 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6e0785e..0768343 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -26,8 +26,14 @@ #pragma warning(disable: 4127) // warning C4127: Conditional expression is constant #endif +/* This will be true on all flat address space platforms and allows us to reduce the + whole npy_intp / size_t / Py_intptr_t business down to just size_t for all size + and dimension types (e.g. shape, strides, indexing), instead of inflicting this + upon the library user. */ +static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); + NAMESPACE_BEGIN(pybind11) -namespace detail { +NAMESPACE_BEGIN(detail) template struct npy_format_descriptor { }; template struct is_pod_struct; @@ -141,10 +147,12 @@ private: return api; } }; -} +NAMESPACE_END(detail) -#define PyArray_GET_(ptr, attr) (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->attr) -#define PyArrayDescr_GET_(ptr, attr) (reinterpret_cast<::pybind11::detail::PyArrayDescr_Proxy*>(ptr)->attr) +#define PyArray_GET_(ptr, attr) \ + (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->attr) +#define PyArrayDescr_GET_(ptr, attr) \ + (reinterpret_cast<::pybind11::detail::PyArrayDescr_Proxy*>(ptr)->attr) #define PyArray_CHKFLAGS_(ptr, flag) \ (flag == (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->flags & flag)) @@ -250,7 +258,7 @@ public: }; array(const pybind11::dtype& dt, const std::vector& shape, - const std::vector& strides, void *ptr = nullptr) { + const std::vector& strides, const void *ptr = nullptr) { auto& api = detail::npy_api::get(); auto ndim = shape.size(); if (shape.size() != strides.size()) @@ -258,7 +266,7 @@ public: auto descr = dt; object tmp(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), - (Py_intptr_t *) strides.data(), ptr, 0, nullptr), false); + (Py_intptr_t *) strides.data(), const_cast(ptr), 0, nullptr), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); if (ptr) @@ -266,20 +274,20 @@ public: m_ptr = tmp.release().ptr(); } - array(const pybind11::dtype& dt, const std::vector& shape, void *ptr = nullptr) + array(const pybind11::dtype& dt, const std::vector& shape, const void *ptr = nullptr) : array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { } - array(const pybind11::dtype& dt, size_t count, void *ptr = nullptr) + array(const pybind11::dtype& dt, size_t count, const void *ptr = nullptr) : array(dt, std::vector { count }, ptr) { } template array(const std::vector& shape, - const std::vector& strides, T* ptr) + const std::vector& strides, const T* ptr) : array(pybind11::dtype::of(), shape, strides, (void *) ptr) { } - template array(const std::vector& shape, T* ptr) + template array(const std::vector& shape, const T* ptr) : array(shape, default_strides(shape, sizeof(T)), ptr) { } - template array(size_t count, T* ptr) + template array(size_t count, const T* ptr) : array(std::vector { count }, ptr) { } array(const buffer_info &info) @@ -312,27 +320,25 @@ public: /// Dimensions of the array const size_t* shape() const { - static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); return reinterpret_cast(PyArray_GET_(m_ptr, dimensions)); } /// Dimension along a given axis size_t shape(size_t dim) const { if (dim >= ndim()) - pybind11_fail("NumPy: attempted to index shape beyond ndim"); + fail_dim_check(dim, "invalid axis"); return shape()[dim]; } /// Strides of the array const size_t* strides() const { - static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); return reinterpret_cast(PyArray_GET_(m_ptr, strides)); } /// Stride along a given axis size_t strides(size_t dim) const { if (dim >= ndim()) - pybind11_fail("NumPy: attempted to index strides beyond ndim"); + fail_dim_check(dim, "invalid axis"); return strides()[dim]; } @@ -346,20 +352,61 @@ public: return PyArray_CHKFLAGS_(m_ptr, detail::npy_api::NPY_ARRAY_OWNDATA_); } - /// Direct pointer to contained buffer - const void* data() const { - return reinterpret_cast(PyArray_GET_(m_ptr, data)); + /// Pointer to the contained data. If index is not provided, points to the + /// beginning of the buffer. May throw if the index would lead to out of bounds access. + template const void* data(Ix&&... index) const { + return static_cast(PyArray_GET_(m_ptr, data) + offset_at(index...)); } - /// Direct mutable pointer to contained buffer (checks writeable flag) - void* mutable_data() { - if (!writeable()) - pybind11_fail("NumPy: cannot get mutable data of a read-only array"); - return reinterpret_cast(PyArray_GET_(m_ptr, data)); + /// Mutable pointer to the contained data. If index is not provided, points to the + /// beginning of the buffer. May throw if the index would lead to out of bounds access. + /// May throw if the array is not writeable. + template void* mutable_data(Ix&&... index) { + check_writeable(); + return static_cast(PyArray_GET_(m_ptr, data) + offset_at(index...)); + } + + /// Byte offset from beginning of the array to a given index (full or partial). + /// May throw if the index would lead to out of bounds access. + template size_t offset_at(Ix&&... index) const { + if (sizeof...(index) > ndim()) + fail_dim_check(sizeof...(index), "too many indices for an array"); + return get_byte_offset(index...); + } + + size_t offset_at() const { return 0; } + + /// Item count from beginning of the array to a given index (full or partial). + /// May throw if the index would lead to out of bounds access. + template size_t index_at(Ix&&... index) const { + return offset_at(index...) / itemsize(); } protected: - template friend struct detail::npy_format_descriptor; + template friend struct detail::npy_format_descriptor; + + void fail_dim_check(size_t dim, const std::string& msg) const { + throw index_error(msg + ": " + std::to_string(dim) + + " (ndim = " + std::to_string(ndim()) + ")"); + } + + template size_t get_byte_offset(Ix&&... index) const { + const size_t idx[] = { (size_t) index... }; + if (!std::equal(idx + 0, idx + sizeof...(index), shape(), std::less{})) { + auto mismatch = std::mismatch(idx + 0, idx + sizeof...(index), shape(), std::less{}); + throw index_error(std::string("index ") + std::to_string(*mismatch.first) + + " is out of bounds for axis " + std::to_string(mismatch.first - idx) + + " with size " + std::to_string(*mismatch.second)); + } + return std::inner_product(idx + 0, idx + sizeof...(index), strides(), (size_t) 0); + } + + size_t get_byte_offset() const { return 0; } + + void check_writeable() const { + if (!writeable()) + throw std::runtime_error("array is not writeable"); + } static std::vector default_strides(const std::vector& shape, size_t itemsize) { auto ndim = shape.size(); @@ -382,23 +429,45 @@ public: array_t(const buffer_info& info) : array(info) { } - array_t(const std::vector& shape, const std::vector& strides, T* ptr = nullptr) + array_t(const std::vector& shape, const std::vector& strides, const T* ptr = nullptr) : array(shape, strides, ptr) { } - array_t(const std::vector& shape, T* ptr = nullptr) + array_t(const std::vector& shape, const T* ptr = nullptr) : array(shape, ptr) { } - array_t(size_t count, T* ptr = nullptr) + array_t(size_t count, const T* ptr = nullptr) : array(count, ptr) { } - const T* data() const { - return reinterpret_cast(PyArray_GET_(m_ptr, data)); + constexpr size_t itemsize() const { + return sizeof(T); } - T* mutable_data() { - if (!writeable()) - pybind11_fail("NumPy: cannot get mutable data of a read-only array"); - return reinterpret_cast(PyArray_GET_(m_ptr, data)); + template size_t index_at(Ix&... index) const { + return offset_at(index...) / itemsize(); + } + + template const T* data(Ix&&... index) const { + return static_cast(array::data(index...)); + } + + template T* mutable_data(Ix&&... index) { + return static_cast(array::mutable_data(index...)); + } + + // Reference to element at a given index + template const T& at(Ix&&... index) const { + if (sizeof...(index) != ndim()) + fail_dim_check(sizeof...(index), "index dimension mismatch"); + // not using offset_at() / index_at() here so as to avoid another dimension check + return *(static_cast(array::data()) + get_byte_offset(index...) / itemsize()); + } + + // Mutable reference to element at a given index + template T& mutable_at(Ix&&... index) { + if (sizeof...(index) != ndim()) + fail_dim_check(sizeof...(index), "index dimension mismatch"); + // not using offset_at() / index_at() here so as to avoid another dimension check + return *(static_cast(array::mutable_data()) + get_byte_offset(index...) / itemsize()); } static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } -- cgit v1.2.3 From c1fc27e2b58391c7b945b06d3e1bfd30e46b62c6 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Tue, 13 Sep 2016 00:36:43 +0900 Subject: use detail::enable_if_t everywhere --- include/pybind11/numpy.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0768343..81dfaea 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -486,7 +486,7 @@ public: }; template -struct format_descriptor::value>::type> { +struct format_descriptor::value>> { static std::string format() { return detail::npy_format_descriptor::type>::format(); } @@ -517,7 +517,7 @@ struct is_pod_struct { !std::is_same::type, std::complex>::value }; }; -template struct npy_format_descriptor::value>::type> { +template struct npy_format_descriptor::value>> { private: constexpr static const int values[8] = { npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, @@ -529,13 +529,13 @@ public: return object(ptr, true); pybind11_fail("Unsupported buffer format!"); } - template ::value, int>::type = 0> + template ::value, int> = 0> static PYBIND11_DESCR name() { return _("int") + _(); } - template ::value, int>::type = 0> + template ::value, int> = 0> static PYBIND11_DESCR name() { return _("uint") + _(); } }; template constexpr const int npy_format_descriptor< - T, typename std::enable_if::value>::type>::values[8]; + T, enable_if_t::value>>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ enum { value = npy_api::NumPyName }; \ @@ -568,7 +568,7 @@ struct field_descriptor { }; template -struct npy_format_descriptor::value>::type> { +struct npy_format_descriptor::value>> { static PYBIND11_DESCR name() { return _("struct"); } static pybind11::dtype dtype() { @@ -634,9 +634,9 @@ private: }; template -std::string npy_format_descriptor::value>::type>::format_str; +std::string npy_format_descriptor::value>>::format_str; template -PyObject* npy_format_descriptor::value>::type>::dtype_ptr = nullptr; +PyObject* npy_format_descriptor::value>>::dtype_ptr = nullptr; // Extract name, offset and format descriptor for a struct field #define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ -- cgit v1.2.3 From c250ee51468abed6fb99c26ed11c0430a4f33ccf Mon Sep 17 00:00:00 2001 From: Dzhelil Rufat Date: Thu, 22 Sep 2016 14:44:11 -0700 Subject: Use more consistent indentation and typenames names. --- include/pybind11/numpy.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 81dfaea..a3e6672 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -911,10 +911,11 @@ detail::vectorize_helper vectorize(Retur return vectorize(f, f); } -template auto vectorize(func &&f) -> decltype( - vectorize(std::forward(f), (typename detail::remove_class::type::operator())>::type *) nullptr)) { - return vectorize(std::forward(f), (typename detail::remove_class::type::operator())>::type *) nullptr); +template +auto vectorize(Func &&f) -> decltype( + vectorize(std::forward(f), (typename detail::remove_class::type::operator())>::type *) nullptr)) { + return vectorize(std::forward(f), (typename detail::remove_class::type::operator())>::type *) nullptr); } NAMESPACE_END(pybind11) -- cgit v1.2.3 From 865e43034b98a317dc523a23075750d564cb477c Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Wed, 21 Sep 2016 01:06:32 +0200 Subject: Make attr and item accessors throw on error instead of returning nullptr This also adds the `hasattr` and `getattr` functions which are needed with the new attribute behavior. The new functions behave exactly like their Python counterparts. Similarly `object` gets a `contains` method which calls `__contains__`, i.e. it's the same as the `in` keyword in Python. --- include/pybind11/numpy.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a3e6672..fb8d3c6 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -125,11 +125,11 @@ private: static npy_api lookup() { module m = module::import("numpy.core.multiarray"); - object c = (object) m.attr("_ARRAY_API"); + auto c = m.attr("_ARRAY_API").cast(); #if PY_MAJOR_VERSION >= 3 - void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr); + void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), NULL); #else - void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); + void **api_ptr = (void **) PyCObject_AsVoidPtr(c.ptr()); #endif npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; -- cgit v1.2.3 From 242b146a5165015045d6547d4840fa6afc6161f4 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Thu, 8 Sep 2016 17:02:04 +0200 Subject: Extend attribute and item accessor interface using object_api --- include/pybind11/numpy.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index fb8d3c6..996bb7c 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -125,7 +125,7 @@ private: static npy_api lookup() { module m = module::import("numpy.core.multiarray"); - auto c = m.attr("_ARRAY_API").cast(); + auto c = m.attr("_ARRAY_API"); #if PY_MAJOR_VERSION >= 3 void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), NULL); #else @@ -220,9 +220,7 @@ private: struct field_descr { PYBIND11_STR_TYPE name; object format; pybind11::int_ offset; }; std::vector field_descriptors; - auto fields = attr("fields").cast(); - auto items = fields.attr("items").cast(); - for (auto field : items()) { + for (auto field : attr("fields").attr("items")()) { auto spec = object(field, true).cast(); auto name = spec[0].cast(); auto format = spec[1].cast()[0].cast(); -- cgit v1.2.3 From ba7678016cf340a3ee733ca3fe941667448eab67 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Fri, 7 Oct 2016 11:19:25 +0200 Subject: numpy.h: added array::squeeze() method --- include/pybind11/numpy.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 996bb7c..1125fd7 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -109,6 +109,7 @@ struct npy_api { bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, Py_ssize_t *, PyObject **, PyObject *); + PyObject *(*PyArray_Squeeze_)(PyObject *); private: enum functions { API_PyArray_Type = 2, @@ -121,6 +122,7 @@ private: API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, API_PyArray_GetArrayParamsFromObject = 278, + API_PyArray_Squeeze = 136 }; static npy_api lookup() { @@ -143,6 +145,7 @@ private: DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_GetArrayParamsFromObject); + DECL_NPY_API(PyArray_Squeeze); #undef DECL_NPY_API return api; } @@ -380,6 +383,12 @@ public: return offset_at(index...) / itemsize(); } + /// Return a new view with all of the dimensions of length 1 removed + array squeeze() { + auto& api = detail::npy_api::get(); + return array(api.PyArray_Squeeze_(m_ptr), false); + } + protected: template friend struct detail::npy_format_descriptor; @@ -601,7 +610,7 @@ struct npy_format_descriptor::value>> { // strings and will just do it ourselves. std::vector ordered_fields(fields); std::sort(ordered_fields.begin(), ordered_fields.end(), - [](const field_descriptor& a, const field_descriptor &b) { + [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); size_t offset = 0; -- cgit v1.2.3 From 369e9b39377dfab610328a1f8de7d83273b82f55 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Thu, 13 Oct 2016 00:57:42 +0200 Subject: Permit creation of NumPy arrays with a "base" object that owns the data This patch adds an extra base handle parameter to most ``py::array`` and ``py::array_t<>`` constructors. If specified along with a pointer to data, the base object will be registered within NumPy, which increases the base's reference count. This feature is useful to create shallow copies of C++ or Python arrays while ensuring that the owners of the underlying can't be garbage collected while referenced by NumPy. The commit also adds a simple test function involving a ``wrap()`` function that creates shallow copies of various N-D arrays. --- include/pybind11/numpy.h | 87 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 24 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1125fd7..445a636 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -22,8 +22,8 @@ #include #if defined(_MSC_VER) -#pragma warning(push) -#pragma warning(disable: 4127) // warning C4127: Conditional expression is constant +# pragma warning(push) +# pragma warning(disable: 4127) // warning C4127: Conditional expression is constant #endif /* This will be true on all flat address space platforms and allows us to reduce the @@ -156,8 +156,10 @@ NAMESPACE_END(detail) (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->attr) #define PyArrayDescr_GET_(ptr, attr) \ (reinterpret_cast<::pybind11::detail::PyArrayDescr_Proxy*>(ptr)->attr) +#define PyArray_FLAGS_(ptr) \ + (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->flags) #define PyArray_CHKFLAGS_(ptr, flag) \ - (flag == (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->flags & flag)) + (flag == (PyArray_FLAGS_(ptr) & flag)) class dtype : public object { public: @@ -258,38 +260,62 @@ public: forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; - array(const pybind11::dtype& dt, const std::vector& shape, - const std::vector& strides, const void *ptr = nullptr) { + array(const pybind11::dtype &dt, const std::vector &shape, + const std::vector &strides, const void *ptr = nullptr, + handle base = handle()) { auto& api = detail::npy_api::get(); auto ndim = shape.size(); if (shape.size() != strides.size()) pybind11_fail("NumPy: shape ndim doesn't match strides ndim"); auto descr = dt; + + int flags = 0; + if (base && ptr) { + array base_array(base, true); + if (base_array.check()) + /* Copy flags from base (except baseship bit) */ + flags = base_array.flags() & ~detail::npy_api::NPY_ARRAY_OWNDATA_; + else + /* Writable by default, easy to downgrade later on if needed */ + flags = detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + object tmp(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), - (Py_intptr_t *) strides.data(), const_cast(ptr), 0, nullptr), false); + (Py_intptr_t *) strides.data(), const_cast(ptr), flags, nullptr), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); - if (ptr) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + if (ptr) { + if (base) { + PyArray_GET_(tmp.ptr(), base) = base.inc_ref().ptr(); + } else { + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + } + } m_ptr = tmp.release().ptr(); } - array(const pybind11::dtype& dt, const std::vector& shape, const void *ptr = nullptr) - : array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { } + array(const pybind11::dtype &dt, const std::vector &shape, + const void *ptr = nullptr, handle base = handle()) + : array(dt, shape, default_strides(shape, dt.itemsize()), ptr, base) { } - array(const pybind11::dtype& dt, size_t count, const void *ptr = nullptr) - : array(dt, std::vector { count }, ptr) { } + array(const pybind11::dtype &dt, size_t count, const void *ptr = nullptr, + handle base = handle()) + : array(dt, std::vector{ count }, ptr, base) { } template array(const std::vector& shape, - const std::vector& strides, const T* ptr) - : array(pybind11::dtype::of(), shape, strides, (void *) ptr) { } + const std::vector& strides, + const T* ptr, handle base = handle()) + : array(pybind11::dtype::of(), shape, strides, (void *) ptr, base) { } - template array(const std::vector& shape, const T* ptr) - : array(shape, default_strides(shape, sizeof(T)), ptr) { } + template + array(const std::vector &shape, const T *ptr, + handle base = handle()) + : array(shape, default_strides(shape, sizeof(T)), ptr, base) { } - template array(size_t count, const T* ptr) - : array(std::vector { count }, ptr) { } + template + array(size_t count, const T *ptr, handle base = handle()) + : array(std::vector{ count }, ptr, base) { } array(const buffer_info &info) : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } @@ -319,6 +345,11 @@ public: return (size_t) PyArray_GET_(m_ptr, nd); } + /// Base object + object base() const { + return object(PyArray_GET_(m_ptr, base), true); + } + /// Dimensions of the array const size_t* shape() const { return reinterpret_cast(PyArray_GET_(m_ptr, dimensions)); @@ -343,6 +374,11 @@ public: return strides()[dim]; } + /// Return the NumPy array flags + int flags() const { + return PyArray_FLAGS_(m_ptr); + } + /// If set, the array is writeable (otherwise the buffer is read-only) bool writeable() const { return PyArray_CHKFLAGS_(m_ptr, detail::npy_api::NPY_ARRAY_WRITEABLE_); @@ -436,14 +472,17 @@ public: array_t(const buffer_info& info) : array(info) { } - array_t(const std::vector& shape, const std::vector& strides, const T* ptr = nullptr) - : array(shape, strides, ptr) { } + array_t(const std::vector &shape, + const std::vector &strides, const T *ptr = nullptr, + handle base = handle()) + : array(shape, strides, ptr, base) { } - array_t(const std::vector& shape, const T* ptr = nullptr) - : array(shape, ptr) { } + array_t(const std::vector &shape, const T *ptr = nullptr, + handle base = handle()) + : array(shape, ptr, base) { } - array_t(size_t count, const T* ptr = nullptr) - : array(count, ptr) { } + array_t(size_t count, const T *ptr = nullptr, handle base = handle()) + : array(count, ptr, base) { } constexpr size_t itemsize() const { return sizeof(T); -- cgit v1.2.3 From fac7c0945801bdc37234a9559bfe86ac2f862059 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Thu, 13 Oct 2016 10:37:52 +0200 Subject: NumPy "base" feature: integrated feedback by @aldanor --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 445a636..a99c72e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -157,7 +157,7 @@ NAMESPACE_END(detail) #define PyArrayDescr_GET_(ptr, attr) \ (reinterpret_cast<::pybind11::detail::PyArrayDescr_Proxy*>(ptr)->attr) #define PyArray_FLAGS_(ptr) \ - (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->flags) + PyArray_GET_(ptr, flags) #define PyArray_CHKFLAGS_(ptr, flag) \ (flag == (PyArray_FLAGS_(ptr) & flag)) -- cgit v1.2.3 From c01a1c1ade0ebeda166d9a070318b0abc6391086 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Fri, 14 Oct 2016 01:08:03 +0200 Subject: added array::ensure() function wrapping PyArray_FromAny This convenience function ensures that a py::object is either a py::array, or the implementation will try to convert it into one. Layout requirements (such as c_style or f_style) can be also be provided. --- include/pybind11/numpy.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a99c72e..d437c92 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -425,6 +425,13 @@ public: return array(api.PyArray_Squeeze_(m_ptr), false); } + /// Ensure that the argument is a NumPy array + static array ensure(object input, int ExtraFlags = 0) { + auto& api = detail::npy_api::get(); + return array(api.PyArray_FromAny_( + input.release().ptr(), nullptr, 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr), false); + } + protected: template friend struct detail::npy_format_descriptor; @@ -466,7 +473,7 @@ protected: template class array_t : public array { public: - PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); + PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure_(m_ptr)); array_t() : array() { } @@ -518,7 +525,7 @@ public: static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } - static PyObject *ensure(PyObject *ptr) { + static PyObject *ensure_(PyObject *ptr) { if (ptr == nullptr) return nullptr; auto& api = detail::npy_api::get(); -- cgit v1.2.3 From 12d76600f8b9092707ff3fe4aab61baf8dc2adb8 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 16 Oct 2016 16:27:42 -0400 Subject: Disable most implicit conversion constructors We have various classes that have non-explicit constructors that accept a single argument, which is implicitly making them implicitly convertible from the argument. In a few cases, this is desirable (e.g. implicit conversion of std::string to py::str, or conversion of double to py::float_); in many others, however, it is unintended (e.g. implicit conversion of size_t to some pre-declared py::array_t type). This disables most of the unwanted implicit conversions by marking them `explicit`, and comments the ones that are deliberately left implicit. --- include/pybind11/numpy.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d437c92..cee40c8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -165,16 +165,16 @@ class dtype : public object { public: PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); - dtype(const buffer_info &info) { + explicit dtype(const buffer_info &info) { dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format))); m_ptr = descr.strip_padding().release().ptr(); } - dtype(std::string format) { + explicit dtype(std::string format) { m_ptr = from_args(pybind11::str(format)).release().ptr(); } - dtype(const char *format) : dtype(std::string(format)) { } + explicit dtype(const char *format) : dtype(std::string(format)) { } dtype(list names, list formats, list offsets, size_t itemsize) { dict args; @@ -317,7 +317,7 @@ public: array(size_t count, const T *ptr, handle base = handle()) : array(std::vector{ count }, ptr, base) { } - array(const buffer_info &info) + explicit array(const buffer_info &info) : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } /// Array descriptor (dtype) @@ -477,18 +477,18 @@ public: array_t() : array() { } - array_t(const buffer_info& info) : array(info) { } + explicit array_t(const buffer_info& info) : array(info) { } array_t(const std::vector &shape, const std::vector &strides, const T *ptr = nullptr, handle base = handle()) : array(shape, strides, ptr, base) { } - array_t(const std::vector &shape, const T *ptr = nullptr, + explicit array_t(const std::vector &shape, const T *ptr = nullptr, handle base = handle()) : array(shape, ptr, base) { } - array_t(size_t count, const T *ptr = nullptr, handle base = handle()) + explicit array_t(size_t count, const T *ptr = nullptr, handle base = handle()) : array(count, ptr, base) { } constexpr size_t itemsize() const { @@ -607,7 +607,7 @@ DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #define DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _(); } \ - static pybind11::dtype dtype() { return std::string("S") + std::to_string(N); } + static pybind11::dtype dtype() { return pybind11::dtype(std::string("S") + std::to_string(N)); } template struct npy_format_descriptor { DECL_CHAR_FMT }; template struct npy_format_descriptor> { DECL_CHAR_FMT }; #undef DECL_CHAR_FMT @@ -883,7 +883,7 @@ struct vectorize_helper { typename std::remove_reference::type f; template - vectorize_helper(T&&f) : f(std::forward(f)) { } + explicit vectorize_helper(T&&f) : f(std::forward(f)) { } object operator()(array_t... args) { return run(args..., typename make_index_sequence::type()); -- cgit v1.2.3 From fb74df50c9165e768feac4a46f7e41e115996d62 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Thu, 20 Oct 2016 12:28:08 +0100 Subject: Implement format/numpy descriptors for enums --- include/pybind11/numpy.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index cee40c8..04001d6 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -552,6 +552,14 @@ template struct format_descriptor> { static std::string format() { return std::to_string(N) + "s"; } }; +template +struct format_descriptor::value>> { + static std::string format() { + return format_descriptor< + typename std::remove_cv::type>::type>::format(); + } +}; + NAMESPACE_BEGIN(detail) template struct is_std_array : std::false_type { }; template struct is_std_array> : std::true_type { }; @@ -563,6 +571,7 @@ struct is_pod_struct { !std::is_array::value && !is_std_array::value && !std::is_integral::value && + !std::is_enum::value && !std::is_same::type, float>::value && !std::is_same::type, double>::value && !std::is_same::type, bool>::value && @@ -612,6 +621,14 @@ template struct npy_format_descriptor { DECL_CHAR_FMT }; template struct npy_format_descriptor> { DECL_CHAR_FMT }; #undef DECL_CHAR_FMT +template struct npy_format_descriptor::value>> { +private: + using base_descr = npy_format_descriptor::type>; +public: + static PYBIND11_DESCR name() { return base_descr::name(); } + static pybind11::dtype dtype() { return base_descr::dtype(); } +}; + struct field_descriptor { const char *name; size_t offset; -- cgit v1.2.3 From ba08db4da5c8fc105714ddfc623db6bd67a4eefd Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Thu, 20 Oct 2016 16:09:10 +0100 Subject: Import a few more numpy extern symbols --- include/pybind11/numpy.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 04001d6..aa93c55 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -103,7 +103,9 @@ struct npy_api { PyObject *(*PyArray_DescrNewFromType_)(int); PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; + PyTypeObject *PyVoidArrType_Type_; PyTypeObject *PyArrayDescr_Type_; + PyObject *(*PyArray_DescrFromScalar_)(PyObject *); PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); int (*PyArray_DescrConverter_) (PyObject *, PyObject **); bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); @@ -114,7 +116,9 @@ private: enum functions { API_PyArray_Type = 2, API_PyArrayDescr_Type = 3, + API_PyVoidArrType_Type = 39, API_PyArray_DescrFromType = 45, + API_PyArray_DescrFromScalar = 57, API_PyArray_FromAny = 69, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, @@ -136,8 +140,10 @@ private: npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_Type); + DECL_NPY_API(PyVoidArrType_Type); DECL_NPY_API(PyArrayDescr_Type); DECL_NPY_API(PyArray_DescrFromType); + DECL_NPY_API(PyArray_DescrFromScalar); DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); -- cgit v1.2.3 From 7bf90e8008fcaba6b03dfbc999610928197174fa Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Thu, 20 Oct 2016 16:11:08 +0100 Subject: Add a direct converter for numpy scalars --- include/pybind11/numpy.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index aa93c55..dba8b7a 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -63,6 +63,14 @@ struct PyArray_Proxy { int flags; }; +struct PyVoidScalarObject_Proxy { + PyObject_VAR_HEAD + char *obval; + PyArrayDescr_Proxy *descr; + int flags; + PyObject *base; +}; + struct npy_api { enum constants { NPY_C_CONTIGUOUS_ = 0x0001, @@ -702,11 +710,29 @@ struct npy_format_descriptor::value>> { auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1)); if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) pybind11_fail("NumPy: invalid buffer descriptor!"); + + register_direct_converter(); } private: static std::string format_str; static PyObject* dtype_ptr; + + static void register_direct_converter() { + auto converter = [=](PyObject *obj, void*& value) { + auto& api = npy_api::get(); + if (!PyObject_TypeCheck(obj, api.PyVoidArrType_Type_)) + return false; + if (auto descr = object(api.PyArray_DescrFromScalar_(obj), false)) { + if (api.PyArray_EquivTypes_(dtype_ptr, descr.ptr())) { + value = ((PyVoidScalarObject_Proxy *) obj)->obval; + return true; + } + } + return false; + }; + get_internals().direct_conversions[std::type_index(typeid(T))].push_back(converter); + } }; template -- cgit v1.2.3 From 7edd72db243e676514d8e036e2688fa96e573343 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Thu, 20 Oct 2016 16:57:12 +0100 Subject: Disallow registering dtypes multiple times --- include/pybind11/numpy.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index dba8b7a..602d703 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -668,6 +668,9 @@ struct npy_format_descriptor::value>> { } static void register_dtype(std::initializer_list fields) { + if (dtype_ptr) + pybind11_fail("NumPy: dtype is already registered"); + list names, formats, offsets; for (auto field : fields) { if (!field.descr) -- cgit v1.2.3 From f70cc112f0c0ea2a3fbac47bc092ce165fb98135 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 22 Oct 2016 10:50:39 +0100 Subject: Make dtype from string ctor accept const ref --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 04001d6..588cc15 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -170,7 +170,7 @@ public: m_ptr = descr.strip_padding().release().ptr(); } - explicit dtype(std::string format) { + explicit dtype(const std::string &format) { m_ptr = from_args(pybind11::str(format)).release().ptr(); } -- cgit v1.2.3 From ef5a38044c515a744b4e8a9f7f697f945f558efe Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 22 Oct 2016 10:51:04 +0100 Subject: A few dtype method docstrings --- include/pybind11/numpy.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 588cc15..b354264 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -185,26 +185,30 @@ public: m_ptr = from_args(args).release().ptr(); } + /// This is essentially the same as calling numpy.dtype(args) in Python. static dtype from_args(object args) { - // This is essentially the same as calling np.dtype() constructor in Python PyObject *ptr = nullptr; if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr) pybind11_fail("NumPy: failed to create structured dtype"); return object(ptr, false); } + /// Return dtype associated with a C++ type. template static dtype of() { return detail::npy_format_descriptor::type>::dtype(); } + /// Size of the data type in bytes. size_t itemsize() const { return (size_t) PyArrayDescr_GET_(m_ptr, elsize); } + /// Returns true for structured data types. bool has_fields() const { return PyArrayDescr_GET_(m_ptr, names) != nullptr; } + /// Single-character type code. char kind() const { return PyArrayDescr_GET_(m_ptr, kind); } -- cgit v1.2.3 From 694269435bc5f24d38d5aa6b252e481406f3d2ac Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 22 Oct 2016 10:51:19 +0100 Subject: Allow implicit casts from literal strings to dtype --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b354264..1dc2d04 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -174,7 +174,7 @@ public: m_ptr = from_args(pybind11::str(format)).release().ptr(); } - explicit dtype(const char *format) : dtype(std::string(format)) { } + dtype(const char *format) : dtype(std::string(format)) { } dtype(list names, list formats, list offsets, size_t itemsize) { dict args; -- cgit v1.2.3 From 43a88f4574269c5d8e621b6e15f0fcfb2fe317a2 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sat, 22 Oct 2016 10:52:05 +0100 Subject: Reraise existing exception if dtype ctor fails --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1dc2d04..4111ccd 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -189,7 +189,7 @@ public: static dtype from_args(object args) { PyObject *ptr = nullptr; if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr) - pybind11_fail("NumPy: failed to create structured dtype"); + throw error_already_set(); return object(ptr, false); } -- cgit v1.2.3 From a6e6a8b108f49f8e990045e020e2058b15dfcf5b Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Sun, 23 Oct 2016 15:27:13 +0100 Subject: Require existing typeinfo for direct conversions This avoid a hashmap lookup since the pointer to the list of direct converters is now cached in the typeinfo. --- include/pybind11/numpy.h | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 602d703..2db3de2 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -721,20 +721,21 @@ private: static std::string format_str; static PyObject* dtype_ptr; - static void register_direct_converter() { - auto converter = [=](PyObject *obj, void*& value) { - auto& api = npy_api::get(); - if (!PyObject_TypeCheck(obj, api.PyVoidArrType_Type_)) - return false; - if (auto descr = object(api.PyArray_DescrFromScalar_(obj), false)) { - if (api.PyArray_EquivTypes_(dtype_ptr, descr.ptr())) { - value = ((PyVoidScalarObject_Proxy *) obj)->obval; - return true; - } - } + static bool direct_converter(PyObject *obj, void*& value) { + auto& api = npy_api::get(); + if (!PyObject_TypeCheck(obj, api.PyVoidArrType_Type_)) return false; - }; - get_internals().direct_conversions[std::type_index(typeid(T))].push_back(converter); + if (auto descr = object(api.PyArray_DescrFromScalar_(obj), false)) { + if (api.PyArray_EquivTypes_(dtype_ptr, descr.ptr())) { + value = ((PyVoidScalarObject_Proxy *) obj)->obval; + return true; + } + } + return false; + } + + static void register_direct_converter() { + get_internals().direct_conversions[std::type_index(typeid(T))].push_back(direct_converter); } }; -- cgit v1.2.3 From e8b50360fe2523e8d490d40dc9a06b36b7e8516a Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Tue, 1 Nov 2016 13:27:35 +0000 Subject: Add dtype binding macro that allows setting names PYBIND11_NUMPY_DTYPE_EX(Type, F1, "N1", F2, "N2", ...) --- include/pybind11/numpy.h | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 4ae3de8..da04c62 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -20,6 +20,7 @@ #include #include #include +#include #if defined(_MSC_VER) # pragma warning(push) @@ -748,14 +749,16 @@ std::string npy_format_descriptor::value>>::form template PyObject* npy_format_descriptor::value>>::dtype_ptr = nullptr; -// Extract name, offset and format descriptor for a struct field -#define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ - ::pybind11::detail::field_descriptor { \ - #Field, offsetof(Type, Field), sizeof(decltype(static_cast(0)->Field)), \ - ::pybind11::format_descriptor(0)->Field)>::format(), \ - ::pybind11::detail::npy_format_descriptor(0)->Field)>::dtype() \ +#define PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, Name) \ + ::pybind11::detail::field_descriptor { \ + Name, offsetof(T, Field), sizeof(decltype(std::declval().Field)), \ + ::pybind11::format_descriptor().Field)>::format(), \ + ::pybind11::detail::npy_format_descriptor().Field)>::dtype() \ } +// Extract name, offset and format descriptor for a struct field +#define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, #Field) + // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz #define PYBIND11_EVAL0(...) __VA_ARGS__ @@ -792,6 +795,27 @@ PyObject* npy_format_descriptor::value>>::dtype_ ::pybind11::detail::npy_format_descriptor::register_dtype \ ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) +#ifdef _MSC_VER +#define PYBIND11_MAP2_LIST_NEXT1(test, next) \ + PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)) +#else +#define PYBIND11_MAP2_LIST_NEXT1(test, next) \ + PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0) +#endif +#define PYBIND11_MAP2_LIST_NEXT(test, next) \ + PYBIND11_MAP2_LIST_NEXT1 (PYBIND11_MAP_GET_END test, next) +#define PYBIND11_MAP2_LIST0(f, t, x1, x2, peek, ...) \ + f(t, x1, x2) PYBIND11_MAP2_LIST_NEXT (peek, PYBIND11_MAP2_LIST1) (f, t, peek, __VA_ARGS__) +#define PYBIND11_MAP2_LIST1(f, t, x1, x2, peek, ...) \ + f(t, x1, x2) PYBIND11_MAP2_LIST_NEXT (peek, PYBIND11_MAP2_LIST0) (f, t, peek, __VA_ARGS__) +// PYBIND11_MAP2_LIST(f, t, a1, a2, ...) expands to f(t, a1, a2), f(t, a3, a4), ... +#define PYBIND11_MAP2_LIST(f, t, ...) \ + PYBIND11_EVAL (PYBIND11_MAP2_LIST1 (f, t, __VA_ARGS__, (), 0)) + +#define PYBIND11_NUMPY_DTYPE_EX(Type, ...) \ + ::pybind11::detail::npy_format_descriptor::register_dtype \ + ({PYBIND11_MAP2_LIST (PYBIND11_FIELD_DESCRIPTOR_EX, Type, __VA_ARGS__)}) + template using array_iterator = typename std::add_pointer::type; -- cgit v1.2.3 From 2184f6d4d64b4631c943b4c92d35bcd849da51ad Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 31 Oct 2016 13:52:32 +0000 Subject: NumPy dtypes are now shared across extensions --- include/pybind11/numpy.h | 78 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 26 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index da04c62..19bff63 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -21,6 +21,7 @@ #include #include #include +#include #if defined(_MSC_VER) # pragma warning(push) @@ -72,6 +73,39 @@ struct PyVoidScalarObject_Proxy { PyObject *base; }; +struct numpy_type_info { + PyObject* dtype_ptr; + std::string format_str; +}; + +struct numpy_internals { + std::unordered_map registered_dtypes; + + template numpy_type_info *get_type_info(bool throw_if_missing = true) { + auto it = registered_dtypes.find(std::type_index(typeid(T))); + if (it != registered_dtypes.end()) + return &(it->second); + if (throw_if_missing) + pybind11_fail(std::string("NumPy type info missing for ") + typeid(T).name()); + return nullptr; + } +}; + +inline PYBIND11_NOINLINE numpy_internals* load_numpy_internals() { + auto& shared_data = detail::get_internals().shared_data; + auto it = shared_data.find("numpy_internals"); + if (it != shared_data.end()) + return (numpy_internals *)it->second; + auto ptr = new numpy_internals(); + shared_data["numpy_internals"] = ptr; + return ptr; +} + +inline numpy_internals& get_numpy_internals() { + static numpy_internals* ptr = load_numpy_internals(); + return *ptr; +} + struct npy_api { enum constants { NPY_C_CONTIGUOUS_ = 0x0001, @@ -661,30 +695,29 @@ struct npy_format_descriptor::value>> { static PYBIND11_DESCR name() { return _("struct"); } static pybind11::dtype dtype() { - if (!dtype_ptr) - pybind11_fail("NumPy: unsupported buffer format!"); - return object(dtype_ptr, true); + return object(dtype_ptr(), true); } static std::string format() { - if (!dtype_ptr) - pybind11_fail("NumPy: unsupported buffer format!"); + static auto format_str = get_numpy_internals().get_type_info(true)->format_str; return format_str; } static void register_dtype(std::initializer_list fields) { - if (dtype_ptr) + auto& numpy_internals = get_numpy_internals(); + if (numpy_internals.get_type_info(false)) pybind11_fail("NumPy: dtype is already registered"); list names, formats, offsets; for (auto field : fields) { if (!field.descr) - pybind11_fail("NumPy: unsupported field dtype"); + pybind11_fail(std::string("NumPy: unsupported field dtype: `") + + field.name + "` @ " + typeid(T).name()); names.append(PYBIND11_STR_TYPE(field.name)); formats.append(field.descr); offsets.append(pybind11::int_(field.offset)); } - dtype_ptr = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); + auto dtype_ptr = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); // There is an existing bug in NumPy (as of v1.11): trailing bytes are // not encoded explicitly into the format string. This will supposedly @@ -695,9 +728,7 @@ struct npy_format_descriptor::value>> { // strings and will just do it ourselves. std::vector ordered_fields(fields); std::sort(ordered_fields.begin(), ordered_fields.end(), - [](const field_descriptor &a, const field_descriptor &b) { - return a.offset < b.offset; - }); + [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); size_t offset = 0; std::ostringstream oss; oss << "T{"; @@ -711,44 +742,39 @@ struct npy_format_descriptor::value>> { if (sizeof(T) > offset) oss << (sizeof(T) - offset) << 'x'; oss << '}'; - format_str = oss.str(); + auto format_str = oss.str(); // Sanity check: verify that NumPy properly parses our buffer format string auto& api = npy_api::get(); - auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1)); + auto arr = array(buffer_info(nullptr, sizeof(T), format_str, 1)); if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) pybind11_fail("NumPy: invalid buffer descriptor!"); - register_direct_converter(); + auto tindex = std::type_index(typeid(T)); + numpy_internals.registered_dtypes[tindex] = { dtype_ptr, format_str }; + get_internals().direct_conversions[tindex].push_back(direct_converter); } private: - static std::string format_str; - static PyObject* dtype_ptr; + static PyObject* dtype_ptr() { + static PyObject* ptr = get_numpy_internals().get_type_info(true)->dtype_ptr; + return ptr; + } static bool direct_converter(PyObject *obj, void*& value) { auto& api = npy_api::get(); if (!PyObject_TypeCheck(obj, api.PyVoidArrType_Type_)) return false; if (auto descr = object(api.PyArray_DescrFromScalar_(obj), false)) { - if (api.PyArray_EquivTypes_(dtype_ptr, descr.ptr())) { + if (api.PyArray_EquivTypes_(dtype_ptr(), descr.ptr())) { value = ((PyVoidScalarObject_Proxy *) obj)->obval; return true; } } return false; } - - static void register_direct_converter() { - get_internals().direct_conversions[std::type_index(typeid(T))].push_back(direct_converter); - } }; -template -std::string npy_format_descriptor::value>>::format_str; -template -PyObject* npy_format_descriptor::value>>::dtype_ptr = nullptr; - #define PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, Name) \ ::pybind11::detail::field_descriptor { \ Name, offsetof(T, Field), sizeof(decltype(std::declval().Field)), \ -- cgit v1.2.3 From 2dbf0297050533e697d065f4b92283e0ecd37581 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 31 Oct 2016 14:11:10 +0000 Subject: Add public shared_data API NumPy internals are stored under "_numpy_internals" key. --- include/pybind11/numpy.h | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 19bff63..b180cb2 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -91,18 +91,14 @@ struct numpy_internals { } }; -inline PYBIND11_NOINLINE numpy_internals* load_numpy_internals() { - auto& shared_data = detail::get_internals().shared_data; - auto it = shared_data.find("numpy_internals"); - if (it != shared_data.end()) - return (numpy_internals *)it->second; - auto ptr = new numpy_internals(); - shared_data["numpy_internals"] = ptr; - return ptr; +inline PYBIND11_NOINLINE void load_numpy_internals(numpy_internals* &ptr) { + ptr = &get_or_create_shared_data("_numpy_internals"); } inline numpy_internals& get_numpy_internals() { - static numpy_internals* ptr = load_numpy_internals(); + static numpy_internals* ptr = nullptr; + if (!ptr) + load_numpy_internals(ptr); return *ptr; } -- cgit v1.2.3 From cc8ff16547fc841d9bac4e8a3624386d70c566e3 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Mon, 31 Oct 2016 16:16:47 +0000 Subject: Move register_dtype() outside of the template (avoid code bloat if possible) --- include/pybind11/numpy.h | 119 ++++++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 53 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b180cb2..af465a1 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -81,14 +81,18 @@ struct numpy_type_info { struct numpy_internals { std::unordered_map registered_dtypes; - template numpy_type_info *get_type_info(bool throw_if_missing = true) { - auto it = registered_dtypes.find(std::type_index(typeid(T))); + numpy_type_info *get_type_info(const std::type_info& tinfo, bool throw_if_missing = true) { + auto it = registered_dtypes.find(std::type_index(tinfo)); if (it != registered_dtypes.end()) return &(it->second); if (throw_if_missing) - pybind11_fail(std::string("NumPy type info missing for ") + typeid(T).name()); + pybind11_fail(std::string("NumPy type info missing for ") + tinfo.name()); return nullptr; } + + template numpy_type_info *get_type_info(bool throw_if_missing = true) { + return get_type_info(typeid(typename std::remove_cv::type), throw_if_missing); + } }; inline PYBIND11_NOINLINE void load_numpy_internals(numpy_internals* &ptr) { @@ -686,6 +690,62 @@ struct field_descriptor { dtype descr; }; +inline PYBIND11_NOINLINE void register_structured_dtype( + const std::initializer_list& fields, + const std::type_info& tinfo, size_t itemsize, + bool (*direct_converter)(PyObject *, void *&)) +{ + auto& numpy_internals = get_numpy_internals(); + if (numpy_internals.get_type_info(tinfo, false)) + pybind11_fail("NumPy: dtype is already registered"); + + list names, formats, offsets; + for (auto field : fields) { + if (!field.descr) + pybind11_fail(std::string("NumPy: unsupported field dtype: `") + + field.name + "` @ " + tinfo.name()); + names.append(PYBIND11_STR_TYPE(field.name)); + formats.append(field.descr); + offsets.append(pybind11::int_(field.offset)); + } + auto dtype_ptr = pybind11::dtype(names, formats, offsets, itemsize).release().ptr(); + + // There is an existing bug in NumPy (as of v1.11): trailing bytes are + // not encoded explicitly into the format string. This will supposedly + // get fixed in v1.12; for further details, see these: + // - https://github.com/numpy/numpy/issues/7797 + // - https://github.com/numpy/numpy/pull/7798 + // Because of this, we won't use numpy's logic to generate buffer format + // strings and will just do it ourselves. + std::vector ordered_fields(fields); + std::sort(ordered_fields.begin(), ordered_fields.end(), + [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); + size_t offset = 0; + std::ostringstream oss; + oss << "T{"; + for (auto& field : ordered_fields) { + if (field.offset > offset) + oss << (field.offset - offset) << 'x'; + // note that '=' is required to cover the case of unaligned fields + oss << '=' << field.format << ':' << field.name << ':'; + offset = field.offset + field.size; + } + if (itemsize > offset) + oss << (itemsize - offset) << 'x'; + oss << '}'; + auto format_str = oss.str(); + + // Sanity check: verify that NumPy properly parses our buffer format string + auto& api = npy_api::get(); + auto arr = array(buffer_info(nullptr, itemsize, format_str, 1)); + if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) + pybind11_fail("NumPy: invalid buffer descriptor!"); + + auto tindex = std::type_index(tinfo); + numpy_internals.registered_dtypes[tindex] = { dtype_ptr, format_str }; + get_internals().direct_conversions[tindex].push_back(direct_converter); +} + template struct npy_format_descriptor::value>> { static PYBIND11_DESCR name() { return _("struct"); } @@ -699,56 +759,9 @@ struct npy_format_descriptor::value>> { return format_str; } - static void register_dtype(std::initializer_list fields) { - auto& numpy_internals = get_numpy_internals(); - if (numpy_internals.get_type_info(false)) - pybind11_fail("NumPy: dtype is already registered"); - - list names, formats, offsets; - for (auto field : fields) { - if (!field.descr) - pybind11_fail(std::string("NumPy: unsupported field dtype: `") + - field.name + "` @ " + typeid(T).name()); - names.append(PYBIND11_STR_TYPE(field.name)); - formats.append(field.descr); - offsets.append(pybind11::int_(field.offset)); - } - auto dtype_ptr = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); - - // There is an existing bug in NumPy (as of v1.11): trailing bytes are - // not encoded explicitly into the format string. This will supposedly - // get fixed in v1.12; for further details, see these: - // - https://github.com/numpy/numpy/issues/7797 - // - https://github.com/numpy/numpy/pull/7798 - // Because of this, we won't use numpy's logic to generate buffer format - // strings and will just do it ourselves. - std::vector ordered_fields(fields); - std::sort(ordered_fields.begin(), ordered_fields.end(), - [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); - size_t offset = 0; - std::ostringstream oss; - oss << "T{"; - for (auto& field : ordered_fields) { - if (field.offset > offset) - oss << (field.offset - offset) << 'x'; - // note that '=' is required to cover the case of unaligned fields - oss << '=' << field.format << ':' << field.name << ':'; - offset = field.offset + field.size; - } - if (sizeof(T) > offset) - oss << (sizeof(T) - offset) << 'x'; - oss << '}'; - auto format_str = oss.str(); - - // Sanity check: verify that NumPy properly parses our buffer format string - auto& api = npy_api::get(); - auto arr = array(buffer_info(nullptr, sizeof(T), format_str, 1)); - if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) - pybind11_fail("NumPy: invalid buffer descriptor!"); - - auto tindex = std::type_index(typeid(T)); - numpy_internals.registered_dtypes[tindex] = { dtype_ptr, format_str }; - get_internals().direct_conversions[tindex].push_back(direct_converter); + static void register_dtype(const std::initializer_list& fields) { + register_structured_dtype(fields, typeid(typename std::remove_cv::type), + sizeof(T), &direct_converter); } private: -- cgit v1.2.3 From cc4efe69c2183914dd3ae644224a741a74612393 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Tue, 8 Nov 2016 10:53:30 +0100 Subject: more code style checks in Travis CI :) --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index af465a1..4120f28 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -693,8 +693,8 @@ struct field_descriptor { inline PYBIND11_NOINLINE void register_structured_dtype( const std::initializer_list& fields, const std::type_info& tinfo, size_t itemsize, - bool (*direct_converter)(PyObject *, void *&)) -{ + bool (*direct_converter)(PyObject *, void *&)) { + auto& numpy_internals = get_numpy_internals(); if (numpy_internals.get_type_info(tinfo, false)) pybind11_fail("NumPy: dtype is already registered"); -- cgit v1.2.3 From 5027c4f95b65dbf76c3f72ce0fcda89716fb5f62 Mon Sep 17 00:00:00 2001 From: Sylvain Corlay Date: Wed, 16 Nov 2016 08:53:37 -0800 Subject: Switch NumPy variadic indexing to per-value arguments (#500) * Also added unsafe version without checks --- include/pybind11/numpy.h | 61 +++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 24 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 4120f28..5309997 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -444,31 +444,31 @@ public: /// Pointer to the contained data. If index is not provided, points to the /// beginning of the buffer. May throw if the index would lead to out of bounds access. - template const void* data(Ix&&... index) const { + template const void* data(Ix... index) const { return static_cast(PyArray_GET_(m_ptr, data) + offset_at(index...)); } /// Mutable pointer to the contained data. If index is not provided, points to the /// beginning of the buffer. May throw if the index would lead to out of bounds access. /// May throw if the array is not writeable. - template void* mutable_data(Ix&&... index) { + template void* mutable_data(Ix... index) { check_writeable(); return static_cast(PyArray_GET_(m_ptr, data) + offset_at(index...)); } /// Byte offset from beginning of the array to a given index (full or partial). /// May throw if the index would lead to out of bounds access. - template size_t offset_at(Ix&&... index) const { + template size_t offset_at(Ix... index) const { if (sizeof...(index) > ndim()) fail_dim_check(sizeof...(index), "too many indices for an array"); - return get_byte_offset(index...); + return byte_offset(size_t(index)...); } size_t offset_at() const { return 0; } /// Item count from beginning of the array to a given index (full or partial). /// May throw if the index would lead to out of bounds access. - template size_t index_at(Ix&&... index) const { + template size_t index_at(Ix... index) const { return offset_at(index...) / itemsize(); } @@ -493,18 +493,16 @@ protected: " (ndim = " + std::to_string(ndim()) + ")"); } - template size_t get_byte_offset(Ix&&... index) const { - const size_t idx[] = { (size_t) index... }; - if (!std::equal(idx + 0, idx + sizeof...(index), shape(), std::less{})) { - auto mismatch = std::mismatch(idx + 0, idx + sizeof...(index), shape(), std::less{}); - throw index_error(std::string("index ") + std::to_string(*mismatch.first) + - " is out of bounds for axis " + std::to_string(mismatch.first - idx) + - " with size " + std::to_string(*mismatch.second)); - } - return std::inner_product(idx + 0, idx + sizeof...(index), strides(), (size_t) 0); + template size_t byte_offset(Ix... index) const { + check_dimensions(index...); + return byte_offset_unsafe(index...); + } + + template size_t byte_offset_unsafe(size_t i, Ix... index) const { + return i * strides()[dim] + byte_offset_unsafe(index...); } - size_t get_byte_offset() const { return 0; } + template size_t byte_offset_unsafe() const { return 0; } void check_writeable() const { if (!writeable()) @@ -522,6 +520,23 @@ protected: } return strides; } + +protected: + + template void check_dimensions(Ix... index) const { + check_dimensions_impl(size_t(0), shape(), size_t(index)...); + } + + void check_dimensions_impl(size_t, const size_t*) const { } + + template void check_dimensions_impl(size_t axis, const size_t* shape, size_t i, Ix... index) const { + if (i >= *shape) { + throw index_error(std::string("index ") + std::to_string(i) + + " is out of bounds for axis " + std::to_string(axis) + + " with size " + std::to_string(*shape)); + } + check_dimensions_impl(axis + 1, shape + 1, index...); + } }; template class array_t : public array { @@ -548,32 +563,30 @@ public: return sizeof(T); } - template size_t index_at(Ix&... index) const { + template size_t index_at(Ix... index) const { return offset_at(index...) / itemsize(); } - template const T* data(Ix&&... index) const { + template const T* data(Ix... index) const { return static_cast(array::data(index...)); } - template T* mutable_data(Ix&&... index) { + template T* mutable_data(Ix... index) { return static_cast(array::mutable_data(index...)); } // Reference to element at a given index - template const T& at(Ix&&... index) const { + template const T& at(Ix... index) const { if (sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); - // not using offset_at() / index_at() here so as to avoid another dimension check - return *(static_cast(array::data()) + get_byte_offset(index...) / itemsize()); + return *(static_cast(array::data()) + byte_offset(size_t(index)...) / itemsize()); } // Mutable reference to element at a given index - template T& mutable_at(Ix&&... index) { + template T& mutable_at(Ix... index) { if (sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); - // not using offset_at() / index_at() here so as to avoid another dimension check - return *(static_cast(array::mutable_data()) + get_byte_offset(index...) / itemsize()); + return *(static_cast(array::mutable_data()) + byte_offset(size_t(index)...) / itemsize()); } static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } -- cgit v1.2.3 From b4498ef44d5b2fbae84726a5d6a3ad011f053922 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Sun, 23 Oct 2016 14:50:08 +0200 Subject: Add py::isinstance(obj) for generalized Python type checking Allows checking the Python types before creating an object instead of after. For example: ```c++ auto l = list(ptr, true); if (l.check()) // ... ``` The above is replaced with: ```c++ if (isinstance(ptr)) { auto l = reinterpret_borrow(ptr); // ... } ``` This deprecates `py::object::check()`. `py::isinstance()` covers the same use case, but it can also check for user-defined types: ```c++ class Pet { ... }; py::class_(...); m.def("is_pet", [](py::object obj) { return py::isinstance(obj); // works as expected }); ``` --- include/pybind11/numpy.h | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5309997..2ffbc5f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -324,10 +324,9 @@ public: int flags = 0; if (base && ptr) { - array base_array(base, true); - if (base_array.check()) + if (isinstance(base)) /* Copy flags from base (except baseship bit) */ - flags = base_array.flags() & ~detail::npy_api::NPY_ARRAY_OWNDATA_; + flags = array(base, true).flags() & ~detail::npy_api::NPY_ARRAY_OWNDATA_; else /* Writable by default, easy to downgrade later on if needed */ flags = detail::npy_api::NPY_ARRAY_WRITEABLE_; @@ -627,6 +626,21 @@ struct format_descriptor::value>> { }; NAMESPACE_BEGIN(detail) +template +struct pyobject_caster> { + using type = array_t; + + bool load(handle src, bool /* convert */) { + value = type(src, true); + return static_cast(value); + } + + static handle cast(const handle &src, return_value_policy /* policy */, handle /* parent */) { + return src.inc_ref(); + } + PYBIND11_TYPE_CASTER(type, handle_type_name::name()); +}; + template struct is_std_array : std::false_type { }; template struct is_std_array> : std::true_type { }; -- cgit v1.2.3 From e18bc02fc9bc17f537aa3d57ebd08c277b6a3959 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Tue, 25 Oct 2016 22:12:39 +0200 Subject: Add default and converting constructors for all concrete Python types * Deprecate the `py::object::str()` member function since `py::str(obj)` is now equivalent and preferred * Make `py::repr()` a free function * Make sure obj.cast() works as expected when T is a Python type `obj.cast()` should be the same as `T(obj)`, i.e. it should convert the given object to a different Python type. However, `obj.cast()` usually calls `type_caster::load()` which only checks the type without doing any actual conversion. That causes a very unexpected `cast_error`. This commit makes it so that `obj.cast()` and `T(obj)` are the same when T is a Python type. * Simplify pytypes converting constructor implementation It's not necessary to maintain a full set of converting constructors and assignment operators + const& and &&. A single converting const& constructor will work and there is no impact on binary size. On the other hand, the conversion functions can be significantly simplified. --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2ffbc5f..9fe4385 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -540,10 +540,12 @@ protected: template class array_t : public array { public: - PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure_(m_ptr)); - array_t() : array() { } + array_t(handle h, bool borrowed) : array(h, borrowed) { m_ptr = ensure_(m_ptr); } + + array_t(const object &o) : array(o) { m_ptr = ensure_(m_ptr); } + explicit array_t(const buffer_info& info) : array(info) { } array_t(const std::vector &shape, @@ -588,8 +590,6 @@ public: return *(static_cast(array::mutable_data()) + byte_offset(size_t(index)...) / itemsize()); } - static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } - static PyObject *ensure_(PyObject *ptr) { if (ptr == nullptr) return nullptr; -- cgit v1.2.3 From c7ac16bb2ea29f8096edb2bf7d7b148c02ef9cba Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Fri, 28 Oct 2016 03:08:15 +0200 Subject: Add py::reinterpret_borrow()/steal() for low-level unchecked casts The pytype converting constructors are convenient and safe for user code, but for library internals the additional type checks and possible conversions are sometimes not desired. `reinterpret_borrow()` and `reinterpret_steal()` serve as the low-level unsafe counterparts of `cast()`. This deprecates the `object(handle, bool)` constructor. Renamed `borrowed` parameter to `is_borrowed` to avoid shadowing warnings on MSVC. --- include/pybind11/numpy.h | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 9fe4385..3cbea01 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -239,7 +239,7 @@ public: PyObject *ptr = nullptr; if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr) throw error_already_set(); - return object(ptr, false); + return reinterpret_steal(ptr); } /// Return dtype associated with a C++ type. @@ -266,7 +266,7 @@ private: static object _dtype_from_pep3118() { static PyObject *obj = module::import("numpy.core._internal") .attr("_dtype_from_pep3118").cast().release().ptr(); - return object(obj, true); + return reinterpret_borrow(obj); } dtype strip_padding() { @@ -279,7 +279,7 @@ private: std::vector field_descriptors; for (auto field : attr("fields").attr("items")()) { - auto spec = object(field, true).cast(); + auto spec = field.cast(); auto name = spec[0].cast(); auto format = spec[1].cast()[0].cast(); auto offset = spec[1].cast()[1].cast(); @@ -326,22 +326,22 @@ public: if (base && ptr) { if (isinstance(base)) /* Copy flags from base (except baseship bit) */ - flags = array(base, true).flags() & ~detail::npy_api::NPY_ARRAY_OWNDATA_; + flags = reinterpret_borrow(base).flags() & ~detail::npy_api::NPY_ARRAY_OWNDATA_; else /* Writable by default, easy to downgrade later on if needed */ flags = detail::npy_api::NPY_ARRAY_WRITEABLE_; } - object tmp(api.PyArray_NewFromDescr_( + auto tmp = reinterpret_steal(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), - (Py_intptr_t *) strides.data(), const_cast(ptr), flags, nullptr), false); + (Py_intptr_t *) strides.data(), const_cast(ptr), flags, nullptr)); if (!tmp) pybind11_fail("NumPy: unable to create array!"); if (ptr) { if (base) { PyArray_GET_(tmp.ptr(), base) = base.inc_ref().ptr(); } else { - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + tmp = reinterpret_steal(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */)); } } m_ptr = tmp.release().ptr(); @@ -374,7 +374,7 @@ public: /// Array descriptor (dtype) pybind11::dtype dtype() const { - return object(PyArray_GET_(m_ptr, descr), true); + return reinterpret_borrow(PyArray_GET_(m_ptr, descr)); } /// Total number of elements @@ -399,7 +399,7 @@ public: /// Base object object base() const { - return object(PyArray_GET_(m_ptr, base), true); + return reinterpret_borrow(PyArray_GET_(m_ptr, base)); } /// Dimensions of the array @@ -474,14 +474,14 @@ public: /// Return a new view with all of the dimensions of length 1 removed array squeeze() { auto& api = detail::npy_api::get(); - return array(api.PyArray_Squeeze_(m_ptr), false); + return reinterpret_steal(api.PyArray_Squeeze_(m_ptr)); } /// Ensure that the argument is a NumPy array static array ensure(object input, int ExtraFlags = 0) { auto& api = detail::npy_api::get(); - return array(api.PyArray_FromAny_( - input.release().ptr(), nullptr, 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr), false); + return reinterpret_steal(api.PyArray_FromAny_( + input.release().ptr(), nullptr, 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr)); } protected: @@ -542,7 +542,7 @@ template class array_t : public public: array_t() : array() { } - array_t(handle h, bool borrowed) : array(h, borrowed) { m_ptr = ensure_(m_ptr); } + array_t(handle h, bool is_borrowed) : array(h, is_borrowed) { m_ptr = ensure_(m_ptr); } array_t(const object &o) : array(o) { m_ptr = ensure_(m_ptr); } @@ -668,7 +668,7 @@ public: enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; static pybind11::dtype dtype() { if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) - return object(ptr, true); + return reinterpret_borrow(ptr); pybind11_fail("Unsupported buffer format!"); } template ::value, int> = 0> @@ -683,7 +683,7 @@ template constexpr const int npy_format_descriptor< enum { value = npy_api::NumPyName }; \ static pybind11::dtype dtype() { \ if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ - return object(ptr, true); \ + return reinterpret_borrow(ptr); \ pybind11_fail("Unsupported buffer format!"); \ } \ static PYBIND11_DESCR name() { return _(Name); } } @@ -778,7 +778,7 @@ struct npy_format_descriptor::value>> { static PYBIND11_DESCR name() { return _("struct"); } static pybind11::dtype dtype() { - return object(dtype_ptr(), true); + return reinterpret_borrow(dtype_ptr()); } static std::string format() { @@ -801,7 +801,7 @@ private: auto& api = npy_api::get(); if (!PyObject_TypeCheck(obj, api.PyVoidArrType_Type_)) return false; - if (auto descr = object(api.PyArray_DescrFromScalar_(obj), false)) { + if (auto descr = reinterpret_steal(api.PyArray_DescrFromScalar_(obj))) { if (api.PyArray_EquivTypes_(dtype_ptr(), descr.ptr())) { value = ((PyVoidScalarObject_Proxy *) obj)->obval; return true; -- cgit v1.2.3 From 4de271027d9e8b1daed2ba111c495cae23c970ac Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Wed, 16 Nov 2016 01:35:22 +0100 Subject: Improve consistency of array and array_t with regard to other pytypes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * `array_t(const object &)` now throws on error * `array_t::ensure()` is intended for casters —- old constructor is deprecated * `array` and `array_t` get default constructors (empty array) * `array` gets a converting constructor * `py::isinstance>()` checks the type (but not flags) There is only one special thing which must remain: `array_t` gets its own `type_caster` specialization which uses `ensure` instead of a simple check. --- include/pybind11/numpy.h | 67 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 18 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3cbea01..77006c8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -305,7 +305,7 @@ private: class array : public buffer { public: - PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) + PYBIND11_OBJECT_CVT(array, buffer, detail::npy_api::get().PyArray_Check_, raw_array) enum { c_style = detail::npy_api::NPY_C_CONTIGUOUS_, @@ -313,6 +313,8 @@ public: forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; + array() : array(0, static_cast(nullptr)) {} + array(const pybind11::dtype &dt, const std::vector &shape, const std::vector &strides, const void *ptr = nullptr, handle base = handle()) { @@ -478,10 +480,12 @@ public: } /// Ensure that the argument is a NumPy array - static array ensure(object input, int ExtraFlags = 0) { - auto& api = detail::npy_api::get(); - return reinterpret_steal(api.PyArray_FromAny_( - input.release().ptr(), nullptr, 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr)); + /// In case of an error, nullptr is returned and the Python error is cleared. + static array ensure(handle h, int ExtraFlags = 0) { + auto result = reinterpret_steal(raw_array(h.ptr(), ExtraFlags)); + if (!result) + PyErr_Clear(); + return result; } protected: @@ -520,8 +524,6 @@ protected: return strides; } -protected: - template void check_dimensions(Ix... index) const { check_dimensions_impl(size_t(0), shape(), size_t(index)...); } @@ -536,15 +538,31 @@ protected: } check_dimensions_impl(axis + 1, shape + 1, index...); } + + /// Create array from any object -- always returns a new reference + static PyObject *raw_array(PyObject *ptr, int ExtraFlags = 0) { + if (ptr == nullptr) + return nullptr; + return detail::npy_api::get().PyArray_FromAny_( + ptr, nullptr, 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + } }; template class array_t : public array { public: - array_t() : array() { } + array_t() : array(0, static_cast(nullptr)) {} + array_t(handle h, borrowed_t) : array(h, borrowed) { } + array_t(handle h, stolen_t) : array(h, stolen) { } - array_t(handle h, bool is_borrowed) : array(h, is_borrowed) { m_ptr = ensure_(m_ptr); } + PYBIND11_DEPRECATED("Use array_t::ensure() instead") + array_t(handle h, bool is_borrowed) : array(raw_array_t(h.ptr()), stolen) { + if (!m_ptr) PyErr_Clear(); + if (!is_borrowed) Py_XDECREF(h.ptr()); + } - array_t(const object &o) : array(o) { m_ptr = ensure_(m_ptr); } + array_t(const object &o) : array(raw_array_t(o.ptr()), stolen) { + if (!m_ptr) throw error_already_set(); + } explicit array_t(const buffer_info& info) : array(info) { } @@ -590,17 +608,30 @@ public: return *(static_cast(array::mutable_data()) + byte_offset(size_t(index)...) / itemsize()); } - static PyObject *ensure_(PyObject *ptr) { - if (ptr == nullptr) - return nullptr; - auto& api = detail::npy_api::get(); - PyObject *result = api.PyArray_FromAny_(ptr, pybind11::dtype::of().release().ptr(), 0, 0, - detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + /// Ensure that the argument is a NumPy array of the correct dtype. + /// In case of an error, nullptr is returned and the Python error is cleared. + static array_t ensure(handle h) { + auto result = reinterpret_steal(raw_array_t(h.ptr())); if (!result) PyErr_Clear(); - Py_DECREF(ptr); return result; } + + static bool _check(handle h) { + const auto &api = detail::npy_api::get(); + return api.PyArray_Check_(h.ptr()) + && api.PyArray_EquivTypes_(PyArray_GET_(h.ptr(), descr), dtype::of().ptr()); + } + +protected: + /// Create array from any object -- always returns a new reference + static PyObject *raw_array_t(PyObject *ptr) { + if (ptr == nullptr) + return nullptr; + return detail::npy_api::get().PyArray_FromAny_( + ptr, dtype::of().release().ptr(), 0, 0, + detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + } }; template @@ -631,7 +662,7 @@ struct pyobject_caster> { using type = array_t; bool load(handle src, bool /* convert */) { - value = type(src, true); + value = type::ensure(src); return static_cast(value); } -- cgit v1.2.3 From b14f065fa97b6e0baf110e93bdb04587dfb87ed5 Mon Sep 17 00:00:00 2001 From: Sylvain Corlay Date: Tue, 22 Nov 2016 01:29:55 -0900 Subject: numpy.h replace macros with functions (#514) --- include/pybind11/numpy.h | 62 +++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 25 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 77006c8..72dd4b3 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -199,16 +199,28 @@ private: return api; } }; -NAMESPACE_END(detail) -#define PyArray_GET_(ptr, attr) \ - (reinterpret_cast<::pybind11::detail::PyArray_Proxy*>(ptr)->attr) -#define PyArrayDescr_GET_(ptr, attr) \ - (reinterpret_cast<::pybind11::detail::PyArrayDescr_Proxy*>(ptr)->attr) -#define PyArray_FLAGS_(ptr) \ - PyArray_GET_(ptr, flags) -#define PyArray_CHKFLAGS_(ptr, flag) \ - (flag == (PyArray_FLAGS_(ptr) & flag)) +inline PyArray_Proxy* array_proxy(void* ptr) { + return reinterpret_cast(ptr); +} + +inline const PyArray_Proxy* array_proxy(const void* ptr) { + return reinterpret_cast(ptr); +} + +inline PyArrayDescr_Proxy* array_descriptor_proxy(PyObject* ptr) { + return reinterpret_cast(ptr); +} + +inline const PyArrayDescr_Proxy* array_descriptor_proxy(const PyObject* ptr) { + return reinterpret_cast(ptr); +} + +inline bool check_flags(const void* ptr, int flag) { + return (flag == (array_proxy(ptr)->flags & flag)); +} + +NAMESPACE_END(detail) class dtype : public object { public: @@ -249,17 +261,17 @@ public: /// Size of the data type in bytes. size_t itemsize() const { - return (size_t) PyArrayDescr_GET_(m_ptr, elsize); + return (size_t) detail::array_descriptor_proxy(m_ptr)->elsize; } /// Returns true for structured data types. bool has_fields() const { - return PyArrayDescr_GET_(m_ptr, names) != nullptr; + return detail::array_descriptor_proxy(m_ptr)->names != nullptr; } /// Single-character type code. char kind() const { - return PyArrayDescr_GET_(m_ptr, kind); + return detail::array_descriptor_proxy(m_ptr)->kind; } private: @@ -341,7 +353,7 @@ public: pybind11_fail("NumPy: unable to create array!"); if (ptr) { if (base) { - PyArray_GET_(tmp.ptr(), base) = base.inc_ref().ptr(); + detail::array_proxy(tmp.ptr())->base = base.inc_ref().ptr(); } else { tmp = reinterpret_steal(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */)); } @@ -376,7 +388,7 @@ public: /// Array descriptor (dtype) pybind11::dtype dtype() const { - return reinterpret_borrow(PyArray_GET_(m_ptr, descr)); + return reinterpret_borrow(detail::array_proxy(m_ptr)->descr); } /// Total number of elements @@ -386,7 +398,7 @@ public: /// Byte size of a single element size_t itemsize() const { - return (size_t) PyArrayDescr_GET_(PyArray_GET_(m_ptr, descr), elsize); + return (size_t) detail::array_descriptor_proxy(detail::array_proxy(m_ptr)->descr)->elsize; } /// Total number of bytes @@ -396,17 +408,17 @@ public: /// Number of dimensions size_t ndim() const { - return (size_t) PyArray_GET_(m_ptr, nd); + return (size_t) detail::array_proxy(m_ptr)->nd; } /// Base object object base() const { - return reinterpret_borrow(PyArray_GET_(m_ptr, base)); + return reinterpret_borrow(detail::array_proxy(m_ptr)->base); } /// Dimensions of the array const size_t* shape() const { - return reinterpret_cast(PyArray_GET_(m_ptr, dimensions)); + return reinterpret_cast(detail::array_proxy(m_ptr)->dimensions); } /// Dimension along a given axis @@ -418,7 +430,7 @@ public: /// Strides of the array const size_t* strides() const { - return reinterpret_cast(PyArray_GET_(m_ptr, strides)); + return reinterpret_cast(detail::array_proxy(m_ptr)->strides); } /// Stride along a given axis @@ -430,23 +442,23 @@ public: /// Return the NumPy array flags int flags() const { - return PyArray_FLAGS_(m_ptr); + return detail::array_proxy(m_ptr)->flags; } /// If set, the array is writeable (otherwise the buffer is read-only) bool writeable() const { - return PyArray_CHKFLAGS_(m_ptr, detail::npy_api::NPY_ARRAY_WRITEABLE_); + return detail::check_flags(m_ptr, detail::npy_api::NPY_ARRAY_WRITEABLE_); } /// If set, the array owns the data (will be freed when the array is deleted) bool owndata() const { - return PyArray_CHKFLAGS_(m_ptr, detail::npy_api::NPY_ARRAY_OWNDATA_); + return detail::check_flags(m_ptr, detail::npy_api::NPY_ARRAY_OWNDATA_); } /// Pointer to the contained data. If index is not provided, points to the /// beginning of the buffer. May throw if the index would lead to out of bounds access. template const void* data(Ix... index) const { - return static_cast(PyArray_GET_(m_ptr, data) + offset_at(index...)); + return static_cast(detail::array_proxy(m_ptr)->data + offset_at(index...)); } /// Mutable pointer to the contained data. If index is not provided, points to the @@ -454,7 +466,7 @@ public: /// May throw if the array is not writeable. template void* mutable_data(Ix... index) { check_writeable(); - return static_cast(PyArray_GET_(m_ptr, data) + offset_at(index...)); + return static_cast(detail::array_proxy(m_ptr)->data + offset_at(index...)); } /// Byte offset from beginning of the array to a given index (full or partial). @@ -620,7 +632,7 @@ public: static bool _check(handle h) { const auto &api = detail::npy_api::get(); return api.PyArray_Check_(h.ptr()) - && api.PyArray_EquivTypes_(PyArray_GET_(h.ptr(), descr), dtype::of().ptr()); + && api.PyArray_EquivTypes_(detail::array_proxy(h.ptr())->descr, dtype::of().ptr()); } protected: -- cgit v1.2.3 From 47681c183d2835cae0e83e76d60f695928c10a52 Mon Sep 17 00:00:00 2001 From: patstew Date: Tue, 22 Nov 2016 11:17:07 +0000 Subject: Only mark unaligned types in buffers (#505) Previously all types are marked unaligned in buffer format strings, now we test for alignment before adding the '=' marker. --- include/pybind11/numpy.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 72dd4b3..2ba7bcc 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -756,6 +756,7 @@ struct field_descriptor { const char *name; size_t offset; size_t size; + size_t alignment; std::string format; dtype descr; }; @@ -796,8 +797,10 @@ inline PYBIND11_NOINLINE void register_structured_dtype( for (auto& field : ordered_fields) { if (field.offset > offset) oss << (field.offset - offset) << 'x'; - // note that '=' is required to cover the case of unaligned fields - oss << '=' << field.format << ':' << field.name << ':'; + // mark unaligned fields with '=' + if (field.offset % field.alignment) + oss << '='; + oss << field.format << ':' << field.name << ':'; offset = field.offset + field.size; } if (itemsize > offset) @@ -857,6 +860,7 @@ private: #define PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, Name) \ ::pybind11::detail::field_descriptor { \ Name, offsetof(T, Field), sizeof(decltype(std::declval().Field)), \ + alignof(decltype(std::declval().Field)), \ ::pybind11::format_descriptor().Field)>::format(), \ ::pybind11::detail::npy_format_descriptor().Field)>::dtype() \ } -- cgit v1.2.3 From 5271576828bfe8288a320ec87950af1a5c7f112b Mon Sep 17 00:00:00 2001 From: Patrick Stewart Date: Tue, 22 Nov 2016 14:56:52 +0000 Subject: Use correct itemsize when constructing a numpy dtype from a buffer_info --- include/pybind11/numpy.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2ba7bcc..8950cb7 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -228,7 +228,8 @@ public: explicit dtype(const buffer_info &info) { dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format))); - m_ptr = descr.strip_padding().release().ptr(); + // If info.itemsize == 0, use the value calculated from the format string + m_ptr = descr.strip_padding(info.itemsize ? info.itemsize : descr.itemsize()).release().ptr(); } explicit dtype(const std::string &format) { @@ -281,7 +282,7 @@ private: return reinterpret_borrow(obj); } - dtype strip_padding() { + dtype strip_padding(size_t itemsize) { // Recursively strip all void fields with empty names that are generated for // padding fields (as of NumPy v1.11). if (!has_fields()) @@ -297,7 +298,7 @@ private: auto offset = spec[1].cast()[1].cast(); if (!len(name) && format.kind() == 'V') continue; - field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(), offset}); + field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(format.itemsize()), offset}); } std::sort(field_descriptors.begin(), field_descriptors.end(), @@ -311,7 +312,7 @@ private: formats.append(descr.format); offsets.append(descr.offset); } - return dtype(names, formats, offsets, itemsize()); + return dtype(names, formats, offsets, itemsize); } }; -- cgit v1.2.3 From 8c85a857474f629d5e7c5ccd740a660098e417bd Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Sun, 27 Nov 2016 20:56:04 +0100 Subject: Use C++14 index_sequence when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newer standard libraries use compiler intrinsics for std::index_sequence which makes it ‘free’. This prevents hitting instantiation limits for recursive templates (-ftemplate-depth). --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8950cb7..e6f4efd 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1077,7 +1077,7 @@ struct vectorize_helper { explicit vectorize_helper(T&&f) : f(std::forward(f)) { } object operator()(array_t... args) { - return run(args..., typename make_index_sequence::type()); + return run(args..., make_index_sequence()); } template object run(array_t&... args, index_sequence index) { -- cgit v1.2.3 From 6e036e78a75276766a8b4cfaf9637c4dd9b6ecd0 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 13 Dec 2016 20:06:41 -0500 Subject: Support binding noexcept function/methods in C++17 When compiling in C++17 mode the noexcept specifier is part of the function type. This causes a failure in pybind11 because, by omitting a noexcept specifier when deducing function return and argument types, we are implicitly making `noexcept(false)` part of the type. This means that functions with `noexcept` fail to match the function templates in cpp_function (and other places), and we get compilation failure (we end up trying to fit it into the lambda function version, which fails since a function pointer has no `operator()`). We can, however, deduce the true/false `B` in noexcept(B), so we don't need to add a whole other set of overloads, but need to deduce the extra argument when under C++17. That will *not* work under pre-C++17, however. This commit adds two macros to fix the problem: under C++17 (with the appropriate feature macro set) they provide an extra `bool NoExceptions` template argument and provide the `noexcept(NoExceptions)` deduced specifier. Under pre-C++17 they expand to nothing. This is needed to compile pybind11 with gcc7 under -std=c++17. --- include/pybind11/numpy.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e6f4efd..7d46f10 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1145,13 +1145,15 @@ template struct handle_type_name> { NAMESPACE_END(detail) -template -detail::vectorize_helper vectorize(const Func &f, Return (*) (Args ...)) { +template +detail::vectorize_helper +vectorize(const Func &f, Return (*) (Args ...) PYBIND11_NOEXCEPT_SPECIFIER) { return detail::vectorize_helper(f); } -template -detail::vectorize_helper vectorize(Return (*f) (Args ...)) { +template +detail::vectorize_helper +vectorize(Return (*f) (Args ...) PYBIND11_NOEXCEPT_SPECIFIER) { return vectorize(f, f); } -- cgit v1.2.3 From 5f07facef55cf93ff0d01106258255f6025d09ba Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Tue, 3 Jan 2017 11:52:05 +0100 Subject: Fix pointer to reference error in type_caster on MSVC (#583) --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 7d46f10..6fecf28 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1140,7 +1140,7 @@ struct vectorize_helper { }; template struct handle_type_name> { - static PYBIND11_DESCR name() { return _("numpy.ndarray[") + type_caster::name() + _("]"); } + static PYBIND11_DESCR name() { return _("numpy.ndarray[") + make_caster::name() + _("]"); } }; NAMESPACE_END(detail) -- cgit v1.2.3 From f7f5bc8e375a381c8e5b810938f05ef14466a4c3 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 31 Jan 2017 11:00:15 -0500 Subject: Numpy: better compilation errors, long double support (#619) * Clarify PYBIND11_NUMPY_DTYPE documentation The current documentation and example reads as though PYBIND11_NUMPY_DTYPE is a declarative macro along the same lines as PYBIND11_DECLARE_HOLDER_TYPE, but it isn't. The changes the documentation and docs example to make it clear that you need to "call" the macro. * Add satisfies_{all,any,none}_of `satisfies_all_of` is a nice legibility-enhanced shortcut for `is_all, Pred2, Pred3>`. * Give better error message for non-POD dtype attempts If you try to use a non-POD data type, you get difficult-to-interpret compilation errors (about ::name() not being a member of an internal pybind11 struct, among others), for which isn't at all obvious what the problem is. This adds a static_assert for such cases. It also changes the base case from an empty struct to the is_pod_struct case by no longer using `enable_if` but instead using a static_assert: thus specializations avoid the base class, POD types work, and non-POD types (and unimplemented POD types like std::array) get a more informative static_assert failure. * Prefix macros with PYBIND11_ numpy.h uses unprefixed macros, which seems undesirable. This prefixes them with PYBIND11_ to match all the other macros in numpy.h (and elsewhere). * Add long double support This adds long double and std::complex support for numpy arrays. This allows some simplification of the code used to generate format descriptors; the new code uses fewer macros, instead putting the code as different templated options; the template conditions end up simpler with this because we are now supporting all basic C++ arithmetic types (and so can use is_arithmetic instead of is_integral + multiple different specializations). In addition to testing that it is indeed working in the test script, it also adds various offset and size calculations there, which fixes the test failures under x86 compilations. --- include/pybind11/numpy.h | 103 ++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 55 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6fecf28..691b194 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -36,8 +36,7 @@ static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); NAMESPACE_BEGIN(pybind11) NAMESPACE_BEGIN(detail) -template struct npy_format_descriptor { }; -template struct is_pod_struct; +template struct npy_format_descriptor; struct PyArrayDescr_Proxy { PyObject_HEAD @@ -220,6 +219,16 @@ inline bool check_flags(const void* ptr, int flag) { return (flag == (array_proxy(ptr)->flags & flag)); } +template struct is_std_array : std::false_type { }; +template struct is_std_array> : std::true_type { }; +template struct is_complex : std::false_type { }; +template struct is_complex> : std::true_type { }; + +template using is_pod_struct = all_of< + std::is_pod, // since we're accessing directly in memory we need a POD type + satisfies_none_of +>; + NAMESPACE_END(detail) class dtype : public object { @@ -685,65 +694,48 @@ struct pyobject_caster> { PYBIND11_TYPE_CASTER(type, handle_type_name::name()); }; -template struct is_std_array : std::false_type { }; -template struct is_std_array> : std::true_type { }; - -template -struct is_pod_struct { - enum { value = std::is_pod::value && // offsetof only works correctly for POD types - !std::is_reference::value && - !std::is_array::value && - !is_std_array::value && - !std::is_integral::value && - !std::is_enum::value && - !std::is_same::type, float>::value && - !std::is_same::type, double>::value && - !std::is_same::type, bool>::value && - !std::is_same::type, std::complex>::value && - !std::is_same::type, std::complex>::value }; -}; - -template struct npy_format_descriptor::value>> { +template struct npy_format_descriptor::value>> { private: - constexpr static const int values[8] = { - npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, - npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ }; + // NB: the order here must match the one in common.h + constexpr static const int values[15] = { + npy_api::NPY_BOOL_, + npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, + npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_, + npy_api::NPY_FLOAT_, npy_api::NPY_DOUBLE_, npy_api::NPY_LONGDOUBLE_, + npy_api::NPY_CFLOAT_, npy_api::NPY_CDOUBLE_, npy_api::NPY_CLONGDOUBLE_ + }; + public: - enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; + static constexpr int value = values[detail::is_fmt_numeric::index]; + static pybind11::dtype dtype() { if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) return reinterpret_borrow(ptr); pybind11_fail("Unsupported buffer format!"); } - template ::value, int> = 0> - static PYBIND11_DESCR name() { return _("int") + _(); } - template ::value, int> = 0> - static PYBIND11_DESCR name() { return _("uint") + _(); } + template ::value, int> = 0> + static PYBIND11_DESCR name() { + return _::value>(_("bool"), + _::value>("int", "uint") + _()); + } + template ::value, int> = 0> + static PYBIND11_DESCR name() { + return _::value || std::is_same::value>( + _("float") + _(), _("longdouble")); + } + template ::value, int> = 0> + static PYBIND11_DESCR name() { + return _::value || std::is_same::value>( + _("complex") + _(), _("longcomplex")); + } }; -template constexpr const int npy_format_descriptor< - T, enable_if_t::value>>::values[8]; - -#define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - enum { value = npy_api::NumPyName }; \ - static pybind11::dtype dtype() { \ - if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ - return reinterpret_borrow(ptr); \ - pybind11_fail("Unsupported buffer format!"); \ - } \ - static PYBIND11_DESCR name() { return _(Name); } } -DECL_FMT(float, NPY_FLOAT_, "float32"); -DECL_FMT(double, NPY_DOUBLE_, "float64"); -DECL_FMT(bool, NPY_BOOL_, "bool"); -DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); -DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); -#undef DECL_FMT - -#define DECL_CHAR_FMT \ + +#define PYBIND11_DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _(); } \ static pybind11::dtype dtype() { return pybind11::dtype(std::string("S") + std::to_string(N)); } -template struct npy_format_descriptor { DECL_CHAR_FMT }; -template struct npy_format_descriptor> { DECL_CHAR_FMT }; -#undef DECL_CHAR_FMT +template struct npy_format_descriptor { PYBIND11_DECL_CHAR_FMT }; +template struct npy_format_descriptor> { PYBIND11_DECL_CHAR_FMT }; +#undef PYBIND11_DECL_CHAR_FMT template struct npy_format_descriptor::value>> { private: @@ -798,9 +790,9 @@ inline PYBIND11_NOINLINE void register_structured_dtype( for (auto& field : ordered_fields) { if (field.offset > offset) oss << (field.offset - offset) << 'x'; - // mark unaligned fields with '=' + // mark unaligned fields with '^' (unaligned native type) if (field.offset % field.alignment) - oss << '='; + oss << '^'; oss << field.format << ':' << field.name << ':'; offset = field.offset + field.size; } @@ -820,8 +812,9 @@ inline PYBIND11_NOINLINE void register_structured_dtype( get_internals().direct_conversions[tindex].push_back(direct_converter); } -template -struct npy_format_descriptor::value>> { +template struct npy_format_descriptor { + static_assert(is_pod_struct::value, "Attempt to use a non-POD or unimplemented POD type as a numpy dtype"); + static PYBIND11_DESCR name() { return _("struct"); } static pybind11::dtype dtype() { -- cgit v1.2.3 From 0defac597700168aa8a571fc5966bafd2bdcea41 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Tue, 7 Feb 2017 00:06:04 +0100 Subject: renamed _check -> check_ (Identifiers starting with underscores are reserved by the standard) Also fixed a typo in a comment. --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 691b194..19facb0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -349,7 +349,7 @@ public: int flags = 0; if (base && ptr) { if (isinstance(base)) - /* Copy flags from base (except baseship bit) */ + /* Copy flags from base (except ownership bit) */ flags = reinterpret_borrow(base).flags() & ~detail::npy_api::NPY_ARRAY_OWNDATA_; else /* Writable by default, easy to downgrade later on if needed */ @@ -639,7 +639,7 @@ public: return result; } - static bool _check(handle h) { + static bool check_(handle h) { const auto &api = detail::npy_api::get(); return api.PyArray_Check_(h.ptr()) && api.PyArray_EquivTypes_(detail::array_proxy(h.ptr())->descr, dtype::of().ptr()); -- cgit v1.2.3 From e15fa9f99a8bc6597ebadeb410c5fc7383478d16 Mon Sep 17 00:00:00 2001 From: Matthew Woehlke Date: Wed, 8 Feb 2017 17:43:08 -0500 Subject: Avoid C-style const casts (#659) * Avoid C-style const casts Replace C-style casts that discard `const` with `const_cast` (and, where necessary, `reinterpret_cast` as well). * Warn about C-style const-discarding casts Change pybind11_enable_warnings to also enable `-Wcast-qual` (warn if a C-style cast discards `const`) by default. The previous commit should have gotten rid of all of these (at least, all the ones that tripped in my build, which included the tests), and this should discourage more from newly appearing. --- include/pybind11/numpy.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 19facb0..50eb682 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -357,8 +357,10 @@ public: } auto tmp = reinterpret_steal(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), - (Py_intptr_t *) strides.data(), const_cast(ptr), flags, nullptr)); + api.PyArray_Type_, descr.release().ptr(), (int) ndim, + reinterpret_cast(const_cast(shape.data())), + reinterpret_cast(const_cast(strides.data())), + const_cast(ptr), flags, nullptr)); if (!tmp) pybind11_fail("NumPy: unable to create array!"); if (ptr) { @@ -382,7 +384,7 @@ public: template array(const std::vector& shape, const std::vector& strides, const T* ptr, handle base = handle()) - : array(pybind11::dtype::of(), shape, strides, (void *) ptr, base) { } + : array(pybind11::dtype::of(), shape, strides, (const void *) ptr, base) { } template array(const std::vector &shape, const T *ptr, -- cgit v1.2.3 From bee8827a98fbb77def8cbe5c7aa4de956667baec Mon Sep 17 00:00:00 2001 From: Sylvain Corlay Date: Tue, 14 Feb 2017 10:55:01 +0100 Subject: Template array constructor (#582) --- include/pybind11/numpy.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 50eb682..af0f5ae 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -337,8 +337,9 @@ public: array() : array(0, static_cast(nullptr)) {} - array(const pybind11::dtype &dt, const std::vector &shape, - const std::vector &strides, const void *ptr = nullptr, + template + array(const pybind11::dtype &dt, const Shape &shape, + const Strides &strides, const void *ptr = nullptr, handle base = handle()) { auto& api = detail::npy_api::get(); auto ndim = shape.size(); @@ -536,7 +537,7 @@ protected: throw std::runtime_error("array is not writeable"); } - static std::vector default_strides(const std::vector& shape, size_t itemsize) { + template static std::vector default_strides(const Shape& shape, size_t itemsize) { auto ndim = shape.size(); std::vector strides(ndim); if (ndim) { -- cgit v1.2.3 From 329d983392a363dd2d33251724d38c9c3878d116 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Tue, 14 Feb 2017 11:25:47 +0100 Subject: Revert "Template array constructor (#582)" This reverts commit bee8827a98fbb77def8cbe5c7aa4de956667baec. --- include/pybind11/numpy.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index af0f5ae..50eb682 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -337,9 +337,8 @@ public: array() : array(0, static_cast(nullptr)) {} - template - array(const pybind11::dtype &dt, const Shape &shape, - const Strides &strides, const void *ptr = nullptr, + array(const pybind11::dtype &dt, const std::vector &shape, + const std::vector &strides, const void *ptr = nullptr, handle base = handle()) { auto& api = detail::npy_api::get(); auto ndim = shape.size(); @@ -537,7 +536,7 @@ protected: throw std::runtime_error("array is not writeable"); } - template static std::vector default_strides(const Shape& shape, size_t itemsize) { + static std::vector default_strides(const std::vector& shape, size_t itemsize) { auto ndim = shape.size(); std::vector strides(ndim); if (ndim) { -- cgit v1.2.3 From 1d7998e333fe98941d9858e8554631848cd7a4b2 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 17 Feb 2017 06:56:41 -0500 Subject: Revert noexcept deduction in favour of better SFINAE on lambda functions (#677) noexcept deduction, added in PR #555, doesn't work with clang's -std=c++1z; and while it works with g++, it isn't entirely clear to me that it is required to work in C++17. What should work, however, is that C++17 allows implicit conversion of a `noexcept(true)` function pointer to a `noexcept(false)` (i.e. default, noexcept-not-specified) function pointer. That was breaking in pybind11 because the cpp_function template used for lambdas provided a better match (i.e. without requiring an implicit conversion), but it then failed. This commit takes a different approach of using SFINAE on the lambda function to prevent it from matching a non-lambda object, which then gets implicit conversion from a `noexcept` function pointer to a `noexcept(false)` function pointer. This much nicer solution also gets rid of the C++17 NOEXCEPT macros, and works in both clang and g++. --- include/pybind11/numpy.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 50eb682..5d78c3a 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1140,23 +1140,22 @@ template struct handle_type_name> { NAMESPACE_END(detail) -template +template detail::vectorize_helper -vectorize(const Func &f, Return (*) (Args ...) PYBIND11_NOEXCEPT_SPECIFIER) { +vectorize(const Func &f, Return (*) (Args ...)) { return detail::vectorize_helper(f); } -template -detail::vectorize_helper -vectorize(Return (*f) (Args ...) PYBIND11_NOEXCEPT_SPECIFIER) { +template +detail::vectorize_helper +vectorize(Return (*f) (Args ...)) { return vectorize(f, f); } -template +template ::type::operator())>::type> auto vectorize(Func &&f) -> decltype( - vectorize(std::forward(f), (typename detail::remove_class::type::operator())>::type *) nullptr)) { - return vectorize(std::forward(f), (typename detail::remove_class::type::operator())>::type *) nullptr); + vectorize(std::forward(f), (FuncType *) nullptr)) { + return vectorize(std::forward(f), (FuncType *) nullptr); } NAMESPACE_END(pybind11) -- cgit v1.2.3 From 88fff9d18910295db739bf5861cb6825b2384ef3 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 16 Jan 2017 20:15:42 -0500 Subject: Change numpy constants to non-deprecated versions A few of pybind's numpy constants are using the numpy-deprecated names (without "ARRAY_" in them); updated our names to be consistent with current numpy code. --- include/pybind11/numpy.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5d78c3a..7a79aa8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -107,11 +107,11 @@ inline numpy_internals& get_numpy_internals() { struct npy_api { enum constants { - NPY_C_CONTIGUOUS_ = 0x0001, - NPY_F_CONTIGUOUS_ = 0x0002, + NPY_ARRAY_C_CONTIGUOUS_ = 0x0001, + NPY_ARRAY_F_CONTIGUOUS_ = 0x0002, NPY_ARRAY_OWNDATA_ = 0x0004, NPY_ARRAY_FORCECAST_ = 0x0010, - NPY_ENSURE_ARRAY_ = 0x0040, + NPY_ARRAY_ENSUREARRAY_ = 0x0040, NPY_ARRAY_ALIGNED_ = 0x0100, NPY_ARRAY_WRITEABLE_ = 0x0400, NPY_BOOL_ = 0, @@ -330,8 +330,8 @@ public: PYBIND11_OBJECT_CVT(array, buffer, detail::npy_api::get().PyArray_Check_, raw_array) enum { - c_style = detail::npy_api::NPY_C_CONTIGUOUS_, - f_style = detail::npy_api::NPY_F_CONTIGUOUS_, + c_style = detail::npy_api::NPY_ARRAY_C_CONTIGUOUS_, + f_style = detail::npy_api::NPY_ARRAY_F_CONTIGUOUS_, forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; @@ -568,7 +568,7 @@ protected: if (ptr == nullptr) return nullptr; return detail::npy_api::get().PyArray_FromAny_( - ptr, nullptr, 0, 0, detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + ptr, nullptr, 0, 0, detail::npy_api::NPY_ARRAY_ENSUREARRAY_ | ExtraFlags, nullptr); } }; @@ -654,7 +654,7 @@ protected: return nullptr; return detail::npy_api::get().PyArray_FromAny_( ptr, dtype::of().release().ptr(), 0, 0, - detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + detail::npy_api::NPY_ARRAY_ENSUREARRAY_ | ExtraFlags, nullptr); } }; -- cgit v1.2.3 From f86dddf7ba45810c97dc2cf8e8cb6ee1e8477307 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 16 Jan 2017 20:22:00 -0500 Subject: array: fix base handling numpy arrays aren't currently properly setting base: by setting `->base` directly, the base doesn't follow what numpy expects and documents (that is, following chained array bases to the root array). This fixes the behaviour by using numpy's PyArray_SetBaseObject to set the base instead, and then updates the tests to reflect the fixed behaviour. --- include/pybind11/numpy.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 7a79aa8..f2c4d9c 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -154,6 +154,7 @@ struct npy_api { int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, Py_ssize_t *, PyObject **, PyObject *); PyObject *(*PyArray_Squeeze_)(PyObject *); + int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); private: enum functions { API_PyArray_Type = 2, @@ -168,7 +169,8 @@ private: API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, API_PyArray_GetArrayParamsFromObject = 278, - API_PyArray_Squeeze = 136 + API_PyArray_Squeeze = 136, + API_PyArray_SetBaseObject = 282 }; static npy_api lookup() { @@ -194,6 +196,7 @@ private: DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_GetArrayParamsFromObject); DECL_NPY_API(PyArray_Squeeze); + DECL_NPY_API(PyArray_SetBaseObject); #undef DECL_NPY_API return api; } @@ -365,7 +368,7 @@ public: pybind11_fail("NumPy: unable to create array!"); if (ptr) { if (base) { - detail::array_proxy(tmp.ptr())->base = base.inc_ref().ptr(); + api.PyArray_SetBaseObject_(tmp.ptr(), base.inc_ref().ptr()); } else { tmp = reinterpret_steal(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */)); } @@ -632,8 +635,8 @@ public: return *(static_cast(array::mutable_data()) + byte_offset(size_t(index)...) / itemsize()); } - /// Ensure that the argument is a NumPy array of the correct dtype. - /// In case of an error, nullptr is returned and the Python error is cleared. + /// Ensure that the argument is a NumPy array of the correct dtype (and if not, try to convert + /// it). In case of an error, nullptr is returned and the Python error is cleared. static array_t ensure(handle h) { auto result = reinterpret_steal(raw_array_t(h.ptr())); if (!result) -- cgit v1.2.3 From fd7517037b0e108d8acef539ce4d33ae3bde02fc Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 20 Jan 2017 13:50:07 -0500 Subject: Change array's writeable exception to a ValueError Numpy raises ValueError when attempting to modify an array, while py::array is raising a RuntimeError. This changes the exception to a std::domain_error, which gets mapped to the expected ValueError in python. --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index f2c4d9c..ea9914a 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -536,7 +536,7 @@ protected: void check_writeable() const { if (!writeable()) - throw std::runtime_error("array is not writeable"); + throw std::domain_error("array is not writeable"); } static std::vector default_strides(const std::vector& shape, size_t itemsize) { -- cgit v1.2.3 From 51439896239570da704cb1328ce12174d3f0308c Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Tue, 28 Feb 2017 18:07:51 +0100 Subject: Fix compilation of Eigen casters with complex scalars --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index ea9914a..29b139b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -731,7 +731,7 @@ public: template ::value, int> = 0> static PYBIND11_DESCR name() { return _::value || std::is_same::value>( - _("complex") + _(), _("longcomplex")); + _("complex") + _(), _("longcomplex")); } }; -- cgit v1.2.3 From c44fe6fda5c7552373942c93847cf9881ac819c1 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 26 Feb 2017 18:03:00 -0500 Subject: array_t overload resolution support This makes array_t respect overload resolution and noconvert by failing to load when `convert = false` if the src isn't already an array of the correct type. --- include/pybind11/numpy.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 29b139b..1223241 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -688,7 +688,9 @@ template struct pyobject_caster> { using type = array_t; - bool load(handle src, bool /* convert */) { + bool load(handle src, bool convert) { + if (!convert && !type::check_(src)) + return false; value = type::ensure(src); return static_cast(value); } -- cgit v1.2.3 From 16afbcef4633b2c038cef26f7aaa0b4e567eed20 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Mon, 13 Mar 2017 19:17:18 +0100 Subject: Improve py::array_t scalar type information (#724) * Add value_type member alias to py::array_t (resolve #632) * Use numpy scalar name in py::array_t function signatures (e.g. float32/64 instead of just float) --- include/pybind11/numpy.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1223241..0faed31 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -577,6 +577,8 @@ protected: template class array_t : public array { public: + using value_type = T; + array_t() : array(0, static_cast(nullptr)) {} array_t(handle h, borrowed_t) : array(h, borrowed) { } array_t(handle h, stolen_t) : array(h, stolen) { } @@ -822,7 +824,7 @@ inline PYBIND11_NOINLINE void register_structured_dtype( template struct npy_format_descriptor { static_assert(is_pod_struct::value, "Attempt to use a non-POD or unimplemented POD type as a numpy dtype"); - static PYBIND11_DESCR name() { return _("struct"); } + static PYBIND11_DESCR name() { return make_caster::name(); } static pybind11::dtype dtype() { return reinterpret_borrow(dtype_ptr()); @@ -1140,7 +1142,9 @@ struct vectorize_helper { }; template struct handle_type_name> { - static PYBIND11_DESCR name() { return _("numpy.ndarray[") + make_caster::name() + _("]"); } + static PYBIND11_DESCR name() { + return _("numpy.ndarray[") + npy_format_descriptor::name() + _("]"); + } }; NAMESPACE_END(detail) -- cgit v1.2.3 From 0b6d08a0081dc50a1bec7937a67043399755ce99 Mon Sep 17 00:00:00 2001 From: Patrick Stewart Date: Mon, 21 Nov 2016 17:40:43 +0000 Subject: Add function for comparing buffer_info formats to types Allows equivalent integral types and numpy dtypes --- include/pybind11/numpy.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0faed31..5a766d4 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -703,6 +703,13 @@ struct pyobject_caster> { PYBIND11_TYPE_CASTER(type, handle_type_name::name()); }; +template +struct compare_buffer_info::value>> { + static bool compare(const buffer_info& b) { + return npy_api::get().PyArray_EquivTypes_(dtype::of().ptr(), dtype(b).ptr()); + } +}; + template struct npy_format_descriptor::value>> { private: // NB: the order here must match the one in common.h -- cgit v1.2.3 From ae5a8f7eb3407bd420734e1780e83ddceef04403 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Wed, 15 Mar 2017 00:57:56 -0300 Subject: Stop forcing c-contiguous in py::vectorize The only part of the vectorize code that actually needs c-contiguous is the "trivial" broadcast; for non-trivial arguments, the code already uses strides properly (and so handles C-style, F-style, neither, slices, etc.) This commit rewrites `broadcast` to additionally check for C-contiguous storage, then takes off the `c_style` flag for the arguments, which will keep the functionality more or less the same, except for no longer requiring an array copy for non-c-contiguous input arrays. Additionally, if we're given a singleton slice (e.g. a[0::4, 0::4] for a 4x4 or smaller array), we no longer fail triviality because the trivial code path never actually uses the strides on a singleton. --- include/pybind11/numpy.h | 65 +++++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 25 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5a766d4..166cbd0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1052,28 +1052,47 @@ private: std::array m_common_iterator; }; +// Populates the shape and number of dimensions for the set of buffers. Returns true if the +// broadcast is "trivial"--that is, has each buffer being either a singleton or a full-size, +// C-contiguous storage buffer. template -bool broadcast(const std::array& buffers, size_t& ndim, std::vector& shape) { +bool broadcast(const std::array &buffers, size_t &ndim, std::vector &shape) { ndim = std::accumulate(buffers.begin(), buffers.end(), size_t(0), [](size_t res, const buffer_info& buf) { return std::max(res, buf.ndim); }); - shape = std::vector(ndim, 1); + shape.clear(); + shape.resize(ndim, 1); + bool trivial_broadcast = true; for (size_t i = 0; i < N; ++i) { + trivial_broadcast = trivial_broadcast && (buffers[i].size == 1 || buffers[i].ndim == ndim); + size_t expect_stride = buffers[i].itemsize; auto res_iter = shape.rbegin(); - bool i_trivial_broadcast = (buffers[i].size == 1) || (buffers[i].ndim == ndim); - for (auto shape_iter = buffers[i].shape.rbegin(); - shape_iter != buffers[i].shape.rend(); ++shape_iter, ++res_iter) { - - if (*res_iter == 1) - *res_iter = *shape_iter; - else if ((*shape_iter != 1) && (*res_iter != *shape_iter)) + auto stride_iter = buffers[i].strides.rbegin(); + auto shape_iter = buffers[i].shape.rbegin(); + while (shape_iter != buffers[i].shape.rend()) { + const auto &dim_size_in = *shape_iter; + auto &dim_size_out = *res_iter; + + // Each input dimension can either be 1 or `n`, but `n` values must match across buffers + if (dim_size_out == 1) + dim_size_out = dim_size_in; + else if (dim_size_in != 1 && dim_size_in != dim_size_out) pybind11_fail("pybind11::vectorize: incompatible size/dimension of inputs!"); - i_trivial_broadcast = i_trivial_broadcast && (*res_iter == *shape_iter); + if (trivial_broadcast && buffers[i].size > 1) { + if (dim_size_in == dim_size_out && expect_stride == *stride_iter) { + expect_stride *= dim_size_in; + ++stride_iter; + } else { + trivial_broadcast = false; + } + } + + ++shape_iter; + ++res_iter; } - trivial_broadcast = trivial_broadcast && i_trivial_broadcast; } return trivial_broadcast; } @@ -1081,18 +1100,17 @@ bool broadcast(const std::array& buffers, size_t& ndim, std::vec template struct vectorize_helper { typename std::remove_reference::type f; + static constexpr size_t N = sizeof...(Args); template explicit vectorize_helper(T&&f) : f(std::forward(f)) { } - object operator()(array_t... args) { - return run(args..., make_index_sequence()); + object operator()(array_t... args) { + return run(args..., make_index_sequence()); } - template object run(array_t&... args, index_sequence index) { + template object run(array_t&... args, index_sequence index) { /* Request buffers from all parameters */ - const size_t N = sizeof...(Args); - std::array buffers {{ args.request()... }}; /* Determine dimensions parameters of output array */ @@ -1112,27 +1130,24 @@ struct vectorize_helper { } if (size == 1) - return cast(f(*((Args *) buffers[Index].ptr)...)); + return cast(f(*reinterpret_cast(buffers[Index].ptr)...)); - array_t result(shape, strides); + array_t result(shape, strides); auto buf = result.request(); auto output = (Return *) buf.ptr; if (trivial_broadcast) { /* Call the function */ - for (size_t i = 0; i < size; ++i) { - output[i] = f((buffers[Index].size == 1 - ? *((Args *) buffers[Index].ptr) - : ((Args *) buffers[Index].ptr)[i])...); - } + for (size_t i = 0; i < size; ++i) + output[i] = f((reinterpret_cast(buffers[Index].ptr)[buffers[Index].size == 1 ? 0 : i])...); } else { - apply_broadcast(buffers, buf, index); + apply_broadcast(buffers, buf, index); } return result; } - template + template void apply_broadcast(const std::array &buffers, buffer_info &output, index_sequence) { using input_iterator = multi_array_iterator; -- cgit v1.2.3 From b0292c1df32d68759c3dfeb81d51954338d418c6 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sat, 18 Mar 2017 21:11:59 -0300 Subject: vectorize: trivial handling for F-order arrays This extends the trivial handling to support trivial handling for Fortran-order arrays (i.e. column major): if inputs aren't all C-contiguous, but *are* all F-contiguous, the resulting array will be F-contiguous and we can do trivial processing. For anything else (e.g. C-contiguous, or inputs requiring non-trivial processing), the result is in (numpy-default) C-contiguous layout. --- include/pybind11/numpy.h | 106 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 31 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 166cbd0..0c703b8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1052,11 +1052,14 @@ private: std::array m_common_iterator; }; -// Populates the shape and number of dimensions for the set of buffers. Returns true if the -// broadcast is "trivial"--that is, has each buffer being either a singleton or a full-size, -// C-contiguous storage buffer. +enum class broadcast_trivial { non_trivial, c_trivial, f_trivial }; + +// Populates the shape and number of dimensions for the set of buffers. Returns a broadcast_trivial +// enum value indicating whether the broadcast is "trivial"--that is, has each buffer being either a +// singleton or a full-size, C-contiguous (`c_trivial`) or Fortran-contiguous (`f_trivial`) storage +// buffer; returns `non_trivial` otherwise. template -bool broadcast(const std::array &buffers, size_t &ndim, std::vector &shape) { +broadcast_trivial broadcast(const std::array &buffers, size_t &ndim, std::vector &shape) { ndim = std::accumulate(buffers.begin(), buffers.end(), size_t(0), [](size_t res, const buffer_info& buf) { return std::max(res, buf.ndim); }); @@ -1064,14 +1067,12 @@ bool broadcast(const std::array &buffers, size_t &ndim, std::vec shape.clear(); shape.resize(ndim, 1); - bool trivial_broadcast = true; + // Figure out the output size, and make sure all input arrays conform (i.e. are either size 1 or + // the full size). for (size_t i = 0; i < N; ++i) { - trivial_broadcast = trivial_broadcast && (buffers[i].size == 1 || buffers[i].ndim == ndim); - size_t expect_stride = buffers[i].itemsize; auto res_iter = shape.rbegin(); - auto stride_iter = buffers[i].strides.rbegin(); - auto shape_iter = buffers[i].shape.rbegin(); - while (shape_iter != buffers[i].shape.rend()) { + auto end = buffers[i].shape.rend(); + for (auto shape_iter = buffers[i].shape.rbegin(); shape_iter != end; ++shape_iter, ++res_iter) { const auto &dim_size_in = *shape_iter; auto &dim_size_out = *res_iter; @@ -1080,21 +1081,54 @@ bool broadcast(const std::array &buffers, size_t &ndim, std::vec dim_size_out = dim_size_in; else if (dim_size_in != 1 && dim_size_in != dim_size_out) pybind11_fail("pybind11::vectorize: incompatible size/dimension of inputs!"); + } + } - if (trivial_broadcast && buffers[i].size > 1) { - if (dim_size_in == dim_size_out && expect_stride == *stride_iter) { - expect_stride *= dim_size_in; - ++stride_iter; - } else { - trivial_broadcast = false; - } + bool trivial_broadcast_c = true; + bool trivial_broadcast_f = true; + for (size_t i = 0; i < N && (trivial_broadcast_c || trivial_broadcast_f); ++i) { + if (buffers[i].size == 1) + continue; + + // Require the same number of dimensions: + if (buffers[i].ndim != ndim) + return broadcast_trivial::non_trivial; + + // Require all dimensions be full-size: + if (!std::equal(buffers[i].shape.cbegin(), buffers[i].shape.cend(), shape.cbegin())) + return broadcast_trivial::non_trivial; + + // Check for C contiguity (but only if previous inputs were also C contiguous) + if (trivial_broadcast_c) { + size_t expect_stride = buffers[i].itemsize; + auto end = buffers[i].shape.crend(); + for (auto shape_iter = buffers[i].shape.crbegin(), stride_iter = buffers[i].strides.crbegin(); + trivial_broadcast_c && shape_iter != end; ++shape_iter, ++stride_iter) { + if (expect_stride == *stride_iter) + expect_stride *= *shape_iter; + else + trivial_broadcast_c = false; } + } - ++shape_iter; - ++res_iter; + // Check for Fortran contiguity (if previous inputs were also F contiguous) + if (trivial_broadcast_f) { + size_t expect_stride = buffers[i].itemsize; + auto end = buffers[i].shape.cend(); + for (auto shape_iter = buffers[i].shape.cbegin(), stride_iter = buffers[i].strides.cbegin(); + trivial_broadcast_f && shape_iter != end; ++shape_iter, ++stride_iter) { + if (expect_stride == *stride_iter) + expect_stride *= *shape_iter; + else + trivial_broadcast_f = false; + } } } - return trivial_broadcast; + + return + trivial_broadcast_c ? broadcast_trivial::c_trivial : + trivial_broadcast_f ? broadcast_trivial::f_trivial : + broadcast_trivial::non_trivial; } template @@ -1116,32 +1150,42 @@ struct vectorize_helper { /* Determine dimensions parameters of output array */ size_t ndim = 0; std::vector shape(0); - bool trivial_broadcast = broadcast(buffers, ndim, shape); + auto trivial = broadcast(buffers, ndim, shape); size_t size = 1; std::vector strides(ndim); if (ndim > 0) { - strides[ndim-1] = sizeof(Return); - for (size_t i = ndim - 1; i > 0; --i) { - strides[i - 1] = strides[i] * shape[i]; - size *= shape[i]; + if (trivial == broadcast_trivial::f_trivial) { + strides[0] = sizeof(Return); + for (size_t i = 1; i < ndim; ++i) { + strides[i] = strides[i - 1] * shape[i - 1]; + size *= shape[i - 1]; + } + size *= shape[ndim - 1]; + } + else { + strides[ndim-1] = sizeof(Return); + for (size_t i = ndim - 1; i > 0; --i) { + strides[i - 1] = strides[i] * shape[i]; + size *= shape[i]; + } + size *= shape[0]; } - size *= shape[0]; } if (size == 1) return cast(f(*reinterpret_cast(buffers[Index].ptr)...)); - array_t result(shape, strides); + array_t result(shape, strides); auto buf = result.request(); auto output = (Return *) buf.ptr; - if (trivial_broadcast) { - /* Call the function */ + /* Call the function */ + if (trivial == broadcast_trivial::non_trivial) { + apply_broadcast(buffers, buf, index); + } else { for (size_t i = 0; i < size; ++i) output[i] = f((reinterpret_cast(buffers[Index].ptr)[buffers[Index].size == 1 ? 0 : i])...); - } else { - apply_broadcast(buffers, buf, index); } return result; -- cgit v1.2.3 From 423a49b8be20c0d4447d5d668232d2ea779159e5 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 19 Mar 2017 01:14:23 -0300 Subject: array: add unchecked access via proxy object This adds bounds-unchecked access to arrays through a `a.unchecked()` method. (For `array_t`, the `Type` template parameter is omitted). The mutable version (which requires the array have the `writeable` flag) is available as `a.mutable_unchecked<...>()`. Specifying the Dimensions as a template parameter allows storage of an std::array; having the strides and sizes stored that way (as opposed to storing a copy of the array's strides/shape pointers) allows the compiler to make significant optimizations of the shape() method that it can't make with a pointer; testing with nested loops of the form: for (size_t i0 = 0; i0 < r.shape(0); i0++) for (size_t i1 = 0; i1 < r.shape(1); i1++) ... r(i0, i1, ...) += 1; over a 10 million element array gives around a 25% speedup (versus using a pointer) for the 1D case, 33% for 2D, and runs more than twice as fast with a 5D array. --- include/pybind11/numpy.h | 127 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 7 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0c703b8..a5f68cc 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -35,6 +35,9 @@ static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); NAMESPACE_BEGIN(pybind11) + +class array; // Forward declaration + NAMESPACE_BEGIN(detail) template struct npy_format_descriptor; @@ -232,6 +235,78 @@ template using is_pod_struct = all_of< satisfies_none_of >; +template size_t byte_offset_unsafe(const Strides &) { return 0; } +template +size_t byte_offset_unsafe(const Strides &strides, size_t i, Ix... index) { + return i * strides[Dim] + byte_offset_unsafe(strides, index...); +} + +/** Proxy class providing unsafe, unchecked const access to array data. This is constructed through + * the `unchecked()` method of `array` or the `unchecked()` method of `array_t`. + */ +template +class unchecked_reference { +protected: + const unsigned char *data_; + // Storing the shape & strides in local variables (i.e. these arrays) allows the compiler to + // make large performance gains on big, nested loops. + std::array shape_, strides_; + + friend class pybind11::array; + unchecked_reference(const void *data, const size_t *shape, const size_t *strides) + : data_{reinterpret_cast(data)} { + for (size_t i = 0; i < Dims; i++) { + shape_[i] = shape[i]; + strides_[i] = strides[i]; + } + } + +public: + /** Unchecked const reference access to data at the given indices. Omiting trailing indices + * is equivalent to specifying them as 0. + */ + template const T& operator()(Ix... index) const { + static_assert(sizeof...(Ix) <= Dims, "Invalid number of indices for unchecked array reference"); + return *reinterpret_cast(data_ + byte_offset_unsafe(strides_, size_t{index}...)); + } + /** Unchecked const reference access to data; this operator only participates if the reference + * is to a 1-dimensional array. When present, this is exactly equivalent to `obj(index)`. + */ + template > + const T &operator[](size_t index) const { return operator()(index); } + + /// Returns the shape (i.e. size) of dimension `dim` + size_t shape(size_t dim) const { return shape_[dim]; } + + /// Returns the number of dimensions of the array + constexpr static size_t ndim() { return Dims; } +}; + +template +class unchecked_mutable_reference : public unchecked_reference { + friend class pybind11::array; + using ConstBase = unchecked_reference; + using ConstBase::ConstBase; +public: + /// Mutable, unchecked access to data at the given indices. + template T& operator()(Ix... index) { + static_assert(sizeof...(Ix) == Dims, "Invalid number of indices for unchecked array reference"); + return const_cast(ConstBase::operator()(index...)); + } + /** Mutable, unchecked access data at the given index; this operator only participates if the + * reference is to a 1-dimensional array. When present, this is exactly equivalent to `obj(index)`. + */ + template > + T &operator[](size_t index) { return operator()(index); } +}; + +template +struct type_caster> { + static_assert(Dim == (size_t) -1 /* always fail */, "unchecked array proxy object is not castable"); +}; +template +struct type_caster> : type_caster> {}; + NAMESPACE_END(detail) class dtype : public object { @@ -500,6 +575,31 @@ public: return offset_at(index...) / itemsize(); } + /** Returns a proxy object that provides access to the array's data without bounds or + * dimensionality checking. Will throw if the array is missing the `writeable` flag. Use with + * care: the array must not be destroyed or reshaped for the duration of the returned object, + * and the caller must take care not to access invalid dimensions or dimension indices. + */ + template detail::unchecked_mutable_reference mutable_unchecked() { + if (ndim() != Dims) + throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + + "; expected " + std::to_string(Dims)); + return detail::unchecked_mutable_reference(mutable_data(), shape(), strides()); + } + + /** Returns a proxy object that provides const access to the array's data without bounds or + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed or + * reshaped for the duration of the returned object, and the caller must take care not to access + * invalid dimensions or dimension indices. + */ + template detail::unchecked_reference unchecked() const { + if (ndim() != Dims) + throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + + "; expected " + std::to_string(Dims)); + return detail::unchecked_reference(data(), shape(), strides()); + } + /// Return a new view with all of the dimensions of length 1 removed array squeeze() { auto& api = detail::npy_api::get(); @@ -525,15 +625,9 @@ protected: template size_t byte_offset(Ix... index) const { check_dimensions(index...); - return byte_offset_unsafe(index...); + return detail::byte_offset_unsafe(strides(), size_t{index}...); } - template size_t byte_offset_unsafe(size_t i, Ix... index) const { - return i * strides()[dim] + byte_offset_unsafe(index...); - } - - template size_t byte_offset_unsafe() const { return 0; } - void check_writeable() const { if (!writeable()) throw std::domain_error("array is not writeable"); @@ -637,6 +731,25 @@ public: return *(static_cast(array::mutable_data()) + byte_offset(size_t(index)...) / itemsize()); } + /** Returns a proxy object that provides access to the array's data without bounds or + * dimensionality checking. Will throw if the array is missing the `writeable` flag. Use with + * care: the array must not be destroyed or reshaped for the duration of the returned object, + * and the caller must take care not to access invalid dimensions or dimension indices. + */ + template detail::unchecked_mutable_reference mutable_unchecked() { + return array::mutable_unchecked(); + } + + /** Returns a proxy object that provides const access to the array's data without bounds or + * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying + * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped + * for the duration of the returned object, and the caller must take care not to access invalid + * dimensions or dimension indices. + */ + template detail::unchecked_reference unchecked() const { + return array::unchecked(); + } + /// Ensure that the argument is a NumPy array of the correct dtype (and if not, try to convert /// it). In case of an error, nullptr is returned and the Python error is cleared. static array_t ensure(handle h) { -- cgit v1.2.3 From 773339f131488819c878ae1a1d5bfbefec629af6 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 20 Mar 2017 17:48:38 -0300 Subject: array-unchecked: add runtime dimension support and array-compatible methods The extends the previous unchecked support with the ability to determine the dimensions at runtime. This incurs a small performance hit when used (versus the compile-time fixed alternative), but is still considerably faster than the full checks on every call that happen with `.at()`/`.mutable_at()`. --- include/pybind11/numpy.h | 96 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 28 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a5f68cc..3227a12 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -242,67 +242,107 @@ size_t byte_offset_unsafe(const Strides &strides, size_t i, Ix... index) { } /** Proxy class providing unsafe, unchecked const access to array data. This is constructed through - * the `unchecked()` method of `array` or the `unchecked()` method of `array_t`. + * the `unchecked()` method of `array` or the `unchecked()` method of `array_t`. `Dims` + * will be -1 for dimensions determined at runtime. */ -template +template class unchecked_reference { protected: + static constexpr bool Dynamic = Dims < 0; const unsigned char *data_; // Storing the shape & strides in local variables (i.e. these arrays) allows the compiler to - // make large performance gains on big, nested loops. - std::array shape_, strides_; + // make large performance gains on big, nested loops, but requires compile-time dimensions + conditional_t> + shape_, strides_; + const size_t dims_; friend class pybind11::array; - unchecked_reference(const void *data, const size_t *shape, const size_t *strides) - : data_{reinterpret_cast(data)} { - for (size_t i = 0; i < Dims; i++) { + // Constructor for compile-time dimensions: + template + unchecked_reference(const void *data, const size_t *shape, const size_t *strides, enable_if_t) + : data_{reinterpret_cast(data)}, dims_{Dims} { + for (size_t i = 0; i < dims_; i++) { shape_[i] = shape[i]; strides_[i] = strides[i]; } } + // Constructor for runtime dimensions: + template + unchecked_reference(const void *data, const size_t *shape, const size_t *strides, enable_if_t dims) + : data_{reinterpret_cast(data)}, shape_{shape}, strides_{strides}, dims_{dims} {} public: - /** Unchecked const reference access to data at the given indices. Omiting trailing indices - * is equivalent to specifying them as 0. + /** Unchecked const reference access to data at the given indices. For a compile-time known + * number of dimensions, this requires the correct number of arguments; for run-time + * dimensionality, this is not checked (and so is up to the caller to use safely). */ - template const T& operator()(Ix... index) const { - static_assert(sizeof...(Ix) <= Dims, "Invalid number of indices for unchecked array reference"); - return *reinterpret_cast(data_ + byte_offset_unsafe(strides_, size_t{index}...)); + template const T &operator()(Ix... index) const { + static_assert(sizeof...(Ix) == Dims || Dynamic, + "Invalid number of indices for unchecked array reference"); + return *reinterpret_cast(data_ + byte_offset_unsafe(strides_, size_t(index)...)); } /** Unchecked const reference access to data; this operator only participates if the reference * is to a 1-dimensional array. When present, this is exactly equivalent to `obj(index)`. */ - template > + template > const T &operator[](size_t index) const { return operator()(index); } + /// Pointer access to the data at the given indices. + template const T *data(Ix... ix) const { return &operator()(size_t(ix)...); } + + /// Returns the item size, i.e. sizeof(T) + constexpr static size_t itemsize() { return sizeof(T); } + /// Returns the shape (i.e. size) of dimension `dim` size_t shape(size_t dim) const { return shape_[dim]; } /// Returns the number of dimensions of the array - constexpr static size_t ndim() { return Dims; } + size_t ndim() const { return dims_; } + + /// Returns the total number of elements in the referenced array, i.e. the product of the shapes + template + enable_if_t size() const { + return std::accumulate(shape_.begin(), shape_.end(), (size_t) 1, std::multiplies()); + } + template + enable_if_t size() const { + return std::accumulate(shape_, shape_ + ndim(), (size_t) 1, std::multiplies()); + } + + /// Returns the total number of bytes used by the referenced data. Note that the actual span in + /// memory may be larger if the referenced array has non-contiguous strides (e.g. for a slice). + size_t nbytes() const { + return size() * itemsize(); + } }; -template +template class unchecked_mutable_reference : public unchecked_reference { friend class pybind11::array; using ConstBase = unchecked_reference; using ConstBase::ConstBase; + using ConstBase::Dynamic; public: /// Mutable, unchecked access to data at the given indices. template T& operator()(Ix... index) { - static_assert(sizeof...(Ix) == Dims, "Invalid number of indices for unchecked array reference"); + static_assert(sizeof...(Ix) == Dims || Dynamic, + "Invalid number of indices for unchecked array reference"); return const_cast(ConstBase::operator()(index...)); } /** Mutable, unchecked access data at the given index; this operator only participates if the - * reference is to a 1-dimensional array. When present, this is exactly equivalent to `obj(index)`. + * reference is to a 1-dimensional array (or has runtime dimensions). When present, this is + * exactly equivalent to `obj(index)`. */ - template > + template > T &operator[](size_t index) { return operator()(index); } + + /// Mutable pointer access to the data at the given indices. + template T *mutable_data(Ix... ix) { return &operator()(size_t(ix)...); } }; template struct type_caster> { - static_assert(Dim == (size_t) -1 /* always fail */, "unchecked array proxy object is not castable"); + static_assert(Dim == 0 && Dim > 0 /* always fail */, "unchecked array proxy object is not castable"); }; template struct type_caster> : type_caster> {}; @@ -580,11 +620,11 @@ public: * care: the array must not be destroyed or reshaped for the duration of the returned object, * and the caller must take care not to access invalid dimensions or dimension indices. */ - template detail::unchecked_mutable_reference mutable_unchecked() { - if (ndim() != Dims) + template detail::unchecked_mutable_reference mutable_unchecked() { + if (Dims >= 0 && ndim() != (size_t) Dims) throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); - return detail::unchecked_mutable_reference(mutable_data(), shape(), strides()); + return detail::unchecked_mutable_reference(mutable_data(), shape(), strides(), ndim()); } /** Returns a proxy object that provides const access to the array's data without bounds or @@ -593,11 +633,11 @@ public: * reshaped for the duration of the returned object, and the caller must take care not to access * invalid dimensions or dimension indices. */ - template detail::unchecked_reference unchecked() const { - if (ndim() != Dims) + template detail::unchecked_reference unchecked() const { + if (Dims >= 0 && ndim() != (size_t) Dims) throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); - return detail::unchecked_reference(data(), shape(), strides()); + return detail::unchecked_reference(data(), shape(), strides(), ndim()); } /// Return a new view with all of the dimensions of length 1 removed @@ -625,7 +665,7 @@ protected: template size_t byte_offset(Ix... index) const { check_dimensions(index...); - return detail::byte_offset_unsafe(strides(), size_t{index}...); + return detail::byte_offset_unsafe(strides(), size_t(index)...); } void check_writeable() const { @@ -736,7 +776,7 @@ public: * care: the array must not be destroyed or reshaped for the duration of the returned object, * and the caller must take care not to access invalid dimensions or dimension indices. */ - template detail::unchecked_mutable_reference mutable_unchecked() { + template detail::unchecked_mutable_reference mutable_unchecked() { return array::mutable_unchecked(); } @@ -746,7 +786,7 @@ public: * for the duration of the returned object, and the caller must take care not to access invalid * dimensions or dimension indices. */ - template detail::unchecked_reference unchecked() const { + template detail::unchecked_reference unchecked() const { return array::unchecked(); } -- cgit v1.2.3 From d6fdafb203560dc3fbb905e065b2b9cb478b0127 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 26 Mar 2017 12:43:22 -0300 Subject: Fix unchecked type caster template Dim type --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3227a12..146dba4 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -340,11 +340,11 @@ public: template T *mutable_data(Ix... ix) { return &operator()(size_t(ix)...); } }; -template +template struct type_caster> { static_assert(Dim == 0 && Dim > 0 /* always fail */, "unchecked array proxy object is not castable"); }; -template +template struct type_caster> : type_caster> {}; NAMESPACE_END(detail) -- cgit v1.2.3 From 6db60cd945febf2bbfd9aa465d96e689c4a66ba6 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 28 Mar 2017 19:23:37 -0300 Subject: Deprecated borrowed/stolen for borrowed_t{}/stolen_t{} (#771) The constexpr static instances can cause linking failures if the compiler doesn't optimize away the reference, as reported in #770. There's no particularly nice way of fixing this in C++11/14: we can't inline definitions to match the declaration aren't permitted for non-templated static variables (C++17 *does* allows "inline" on variables, but that obviously doesn't help us.) One solution that could work around it is to add an extra inherited subclass to `object`'s hierarchy, but that's a bit of a messy solution and was decided against in #771 in favour of just deprecating (and eventually dropping) the constexpr statics. Fixes #770. --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 146dba4..f8ce14e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -714,16 +714,16 @@ public: using value_type = T; array_t() : array(0, static_cast(nullptr)) {} - array_t(handle h, borrowed_t) : array(h, borrowed) { } - array_t(handle h, stolen_t) : array(h, stolen) { } + array_t(handle h, borrowed_t) : array(h, borrowed_t{}) { } + array_t(handle h, stolen_t) : array(h, stolen_t{}) { } PYBIND11_DEPRECATED("Use array_t::ensure() instead") - array_t(handle h, bool is_borrowed) : array(raw_array_t(h.ptr()), stolen) { + array_t(handle h, bool is_borrowed) : array(raw_array_t(h.ptr()), stolen_t{}) { if (!m_ptr) PyErr_Clear(); if (!is_borrowed) Py_XDECREF(h.ptr()); } - array_t(const object &o) : array(raw_array_t(o.ptr()), stolen) { + array_t(const object &o) : array(raw_array_t(o.ptr()), stolen_t{}) { if (!m_ptr) throw error_already_set(); } -- cgit v1.2.3 From 5749b5023936e23d0d2ae77bd776e1f519e2e412 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 10 Apr 2017 11:05:26 -0400 Subject: array: set exception message on failure When attempting to get a raw array pointer we return nullptr if given a nullptr, which triggers an error_already_set(), but we haven't set an exception message, which results in "Unknown internal error". Callers that want explicit allowing of a nullptr here already handle it (by clearing the exception after the call). --- include/pybind11/numpy.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index f8ce14e..ea64542 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -702,8 +702,10 @@ protected: /// Create array from any object -- always returns a new reference static PyObject *raw_array(PyObject *ptr, int ExtraFlags = 0) { - if (ptr == nullptr) + if (ptr == nullptr) { + PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array from a nullptr"); return nullptr; + } return detail::npy_api::get().PyArray_FromAny_( ptr, nullptr, 0, 0, detail::npy_api::NPY_ARRAY_ENSUREARRAY_ | ExtraFlags, nullptr); } @@ -808,8 +810,10 @@ public: protected: /// Create array from any object -- always returns a new reference static PyObject *raw_array_t(PyObject *ptr) { - if (ptr == nullptr) + if (ptr == nullptr) { + PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array_t from a nullptr"); return nullptr; + } return detail::npy_api::get().PyArray_FromAny_( ptr, dtype::of().release().ptr(), 0, 0, detail::npy_api::NPY_ARRAY_ENSUREARRAY_ | ExtraFlags, nullptr); -- cgit v1.2.3 From 5f38386293f989db87545ce8a619e55085068060 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 7 Apr 2017 15:49:54 -0400 Subject: Accept abitrary containers and iterators for shape/strides This adds support for constructing `buffer_info` and `array`s using arbitrary containers or iterator pairs instead of requiring a vector. This is primarily needed by PR #782 (which makes strides signed to properly support negative strides, and will likely also make shape and itemsize to avoid mixed integer issues), but also needs to preserve backwards compatibility with 2.1 and earlier which accepts the strides parameter as a vector of size_t's. Rather than adding nearly duplicate constructors for each stride-taking constructor, it seems nicer to simply allow any type of container (or iterator pairs). This works by replacing the existing vector arguments with a new `detail::any_container` class that handles implicit conversion of arbitrary containers into a vector of the desired type. It can also be explicitly instantiated with a pair of iterators (e.g. by passing {begin, end} instead of the container). --- include/pybind11/numpy.h | 59 ++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 30 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index ea64542..5d44539 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -455,12 +455,18 @@ public: array() : array(0, static_cast(nullptr)) {} - array(const pybind11::dtype &dt, const std::vector &shape, - const std::vector &strides, const void *ptr = nullptr, - handle base = handle()) { - auto& api = detail::npy_api::get(); - auto ndim = shape.size(); - if (shape.size() != strides.size()) + using ShapeContainer = detail::any_container; + using StridesContainer = detail::any_container; + + // Constructs an array taking shape/strides from arbitrary container types + array(const pybind11::dtype &dt, ShapeContainer shape, StridesContainer strides, + const void *ptr = nullptr, handle base = handle()) { + + if (strides->empty()) + strides = default_strides(*shape, dt.itemsize()); + + auto ndim = shape->size(); + if (ndim != strides->size()) pybind11_fail("NumPy: shape ndim doesn't match strides ndim"); auto descr = dt; @@ -474,10 +480,9 @@ public: flags = detail::npy_api::NPY_ARRAY_WRITEABLE_; } + auto &api = detail::npy_api::get(); auto tmp = reinterpret_steal(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr.release().ptr(), (int) ndim, - reinterpret_cast(const_cast(shape.data())), - reinterpret_cast(const_cast(strides.data())), + api.PyArray_Type_, descr.release().ptr(), (int) ndim, shape->data(), strides->data(), const_cast(ptr), flags, nullptr)); if (!tmp) pybind11_fail("NumPy: unable to create array!"); @@ -491,27 +496,24 @@ public: m_ptr = tmp.release().ptr(); } - array(const pybind11::dtype &dt, const std::vector &shape, - const void *ptr = nullptr, handle base = handle()) - : array(dt, shape, default_strides(shape, dt.itemsize()), ptr, base) { } + array(const pybind11::dtype &dt, ShapeContainer shape, const void *ptr = nullptr, handle base = handle()) + : array(dt, std::move(shape), {}, ptr, base) { } array(const pybind11::dtype &dt, size_t count, const void *ptr = nullptr, handle base = handle()) - : array(dt, std::vector{ count }, ptr, base) { } + : array(dt, ShapeContainer{{ count }}, ptr, base) { } - template array(const std::vector& shape, - const std::vector& strides, - const T* ptr, handle base = handle()) - : array(pybind11::dtype::of(), shape, strides, (const void *) ptr, base) { } + template + array(ShapeContainer shape, StridesContainer strides, const T *ptr, handle base = handle()) + : array(pybind11::dtype::of(), std::move(shape), std::move(strides), ptr, base) { } template - array(const std::vector &shape, const T *ptr, - handle base = handle()) - : array(shape, default_strides(shape, sizeof(T)), ptr, base) { } + array(ShapeContainer shape, const T *ptr, handle base = handle()) + : array(std::move(shape), {}, ptr, base) { } template array(size_t count, const T *ptr, handle base = handle()) - : array(std::vector{ count }, ptr, base) { } + : array({{ count }}, ptr, base) { } explicit array(const buffer_info &info) : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } @@ -673,9 +675,9 @@ protected: throw std::domain_error("array is not writeable"); } - static std::vector default_strides(const std::vector& shape, size_t itemsize) { + static std::vector default_strides(const std::vector& shape, size_t itemsize) { auto ndim = shape.size(); - std::vector strides(ndim); + std::vector strides(ndim); if (ndim) { std::fill(strides.begin(), strides.end(), itemsize); for (size_t i = 0; i < ndim - 1; i++) @@ -731,14 +733,11 @@ public: explicit array_t(const buffer_info& info) : array(info) { } - array_t(const std::vector &shape, - const std::vector &strides, const T *ptr = nullptr, - handle base = handle()) - : array(shape, strides, ptr, base) { } + array_t(ShapeContainer shape, StridesContainer strides, const T *ptr = nullptr, handle base = handle()) + : array(std::move(shape), std::move(strides), ptr, base) { } - explicit array_t(const std::vector &shape, const T *ptr = nullptr, - handle base = handle()) - : array(shape, ptr, base) { } + explicit array_t(ShapeContainer shape, const T *ptr = nullptr, handle base = handle()) + : array(std::move(shape), ptr, base) { } explicit array_t(size_t count, const T *ptr = nullptr, handle base = handle()) : array(count, ptr, base) { } -- cgit v1.2.3 From 201796d94f29a6ab25628505c44c036d2dc43cad Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Wed, 12 Apr 2017 18:35:46 -0400 Subject: Make any_container implicitly constructible from arithmetic values This further reduces the constructors required in buffer_info/numpy by removing the need for the constructors that take a single size_t and just forward it on via an initializer_list to the container-accepting constructor. Unfortunately, in `array` one of the constructors runs into an ambiguity problem with the deprecated `array(handle, bool)` constructor (because both the bool constructor and the any_container constructor involve an implicit conversion, so neither has precedence), so a forwarding constructor is kept there (until the deprecated constructor is eventually removed). --- include/pybind11/numpy.h | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5d44539..b32c388 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -463,7 +463,7 @@ public: const void *ptr = nullptr, handle base = handle()) { if (strides->empty()) - strides = default_strides(*shape, dt.itemsize()); + *strides = default_strides(*shape, dt.itemsize()); auto ndim = shape->size(); if (ndim != strides->size()) @@ -499,9 +499,12 @@ public: array(const pybind11::dtype &dt, ShapeContainer shape, const void *ptr = nullptr, handle base = handle()) : array(dt, std::move(shape), {}, ptr, base) { } - array(const pybind11::dtype &dt, size_t count, const void *ptr = nullptr, - handle base = handle()) - : array(dt, ShapeContainer{{ count }}, ptr, base) { } + // This constructor is only needed to avoid ambiguity with the deprecated (handle, bool) + // constructor that comes from PYBIND11_OBJECT_CVT; once that is gone, the above constructor can + // handle it (because ShapeContainer is implicitly constructible from arithmetic types) + template ::value && !std::is_same::value>> + array(const pybind11::dtype &dt, T count) + : array(dt, count, nullptr) { } template array(ShapeContainer shape, StridesContainer strides, const T *ptr, handle base = handle()) @@ -511,10 +514,6 @@ public: array(ShapeContainer shape, const T *ptr, handle base = handle()) : array(std::move(shape), {}, ptr, base) { } - template - array(size_t count, const T *ptr, handle base = handle()) - : array({{ count }}, ptr, base) { } - explicit array(const buffer_info &info) : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } @@ -739,9 +738,6 @@ public: explicit array_t(ShapeContainer shape, const T *ptr = nullptr, handle base = handle()) : array(std::move(shape), ptr, base) { } - explicit array_t(size_t count, const T *ptr = nullptr, handle base = handle()) - : array(count, ptr, base) { } - constexpr size_t itemsize() const { return sizeof(T); } -- cgit v1.2.3 From ce494d65de445bdfbf397f71ce0979e6e1dc249e Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 28 Apr 2017 08:57:13 -0400 Subject: Add numpy version check (#819) The numpy API constants can check past the end of the API array if the numpy version is too old thus causing a segfault. The current list of functions requires numpy >= 1.7.0, so this adds a check and exception if numpy is too old. The added feature version API element was added in numpy 1.4.0, so this could still segfault if loaded in 1.3.0 or earlier, but given that 1.4.0 was released at the end of 2009, it seems reasonable enough to not worry about that case. (1.7.0 was released in early 2013). --- include/pybind11/numpy.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b32c388..cbf4c2f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -141,6 +141,7 @@ struct npy_api { return (bool) PyObject_TypeCheck(obj, PyArrayDescr_Type_); } + unsigned int (*PyArray_GetNDArrayCFeatureVersion_)(); PyObject *(*PyArray_DescrFromType_)(int); PyObject *(*PyArray_NewFromDescr_) (PyTypeObject *, PyObject *, int, Py_intptr_t *, @@ -160,6 +161,7 @@ struct npy_api { int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); private: enum functions { + API_PyArray_GetNDArrayCFeatureVersion = 211, API_PyArray_Type = 2, API_PyArrayDescr_Type = 3, API_PyVoidArrType_Type = 39, @@ -186,6 +188,9 @@ private: #endif npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; + DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion); + if (api.PyArray_GetNDArrayCFeatureVersion_() < 0x7) + pybind11_fail("pybind11 numpy support requires numpy >= 1.7.0"); DECL_NPY_API(PyArray_Type); DECL_NPY_API(PyVoidArrType_Type); DECL_NPY_API(PyArrayDescr_Type); -- cgit v1.2.3 From 51d18aa252332e883c61b7cb45ed317146b372ce Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 28 Apr 2017 11:06:16 -0400 Subject: Fix ambiguous initialize_list arguments This removes the convert-from-arithemtic-scalar constructor of any_container as it can result in ambiguous calls, as in: py::array_t({ 1, 2 }) which could be intepreted as either of: py::array_t(py::array_t(1, 2)) py::array_t(py::detail::any_container({ 1, 2 })) Removing the convert-from-arithmetic constructor reduces the number of implicit conversions, avoiding the ambiguity for array and array_t. This also re-adds the array/array_t constructors taking a scalar argument for backwards compatibility. --- include/pybind11/numpy.h | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index cbf4c2f..44ea427 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -458,7 +458,7 @@ public: forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; - array() : array(0, static_cast(nullptr)) {} + array() : array({{0}}, static_cast(nullptr)) {} using ShapeContainer = detail::any_container; using StridesContainer = detail::any_container; @@ -504,12 +504,9 @@ public: array(const pybind11::dtype &dt, ShapeContainer shape, const void *ptr = nullptr, handle base = handle()) : array(dt, std::move(shape), {}, ptr, base) { } - // This constructor is only needed to avoid ambiguity with the deprecated (handle, bool) - // constructor that comes from PYBIND11_OBJECT_CVT; once that is gone, the above constructor can - // handle it (because ShapeContainer is implicitly constructible from arithmetic types) - template ::value && !std::is_same::value>> - array(const pybind11::dtype &dt, T count) - : array(dt, count, nullptr) { } + template ::value && !std::is_same::value>> + array(const pybind11::dtype &dt, T count, const void *ptr = nullptr, handle base = handle()) + : array(dt, {{count}}, ptr, base) { } template array(ShapeContainer shape, StridesContainer strides, const T *ptr, handle base = handle()) @@ -519,6 +516,9 @@ public: array(ShapeContainer shape, const T *ptr, handle base = handle()) : array(std::move(shape), {}, ptr, base) { } + template + explicit array(size_t count, const T *ptr, handle base = handle()) : array({count}, {}, ptr, base) { } + explicit array(const buffer_info &info) : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } @@ -743,6 +743,9 @@ public: explicit array_t(ShapeContainer shape, const T *ptr = nullptr, handle base = handle()) : array(std::move(shape), ptr, base) { } + explicit array_t(size_t count, const T *ptr = nullptr, handle base = handle()) + : array({count}, {}, ptr, base) { } + constexpr size_t itemsize() const { return sizeof(T); } -- cgit v1.2.3 From 083a0219b5962a146a5e1556d7ce5e5187cb8bca Mon Sep 17 00:00:00 2001 From: uentity Date: Thu, 13 Apr 2017 21:41:55 +0500 Subject: array: implement array resize --- include/pybind11/numpy.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 44ea427..ba9402b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -129,6 +129,11 @@ struct npy_api { NPY_STRING_, NPY_UNICODE_, NPY_VOID_ }; + typedef struct { + Py_intptr_t *ptr; + int len; + } PyArray_Dims; + static npy_api& get() { static npy_api api = lookup(); return api; @@ -159,6 +164,7 @@ struct npy_api { Py_ssize_t *, PyObject **, PyObject *); PyObject *(*PyArray_Squeeze_)(PyObject *); int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); + PyObject* (*PyArray_Resize_)(PyObject*, PyArray_Dims*, int, int); private: enum functions { API_PyArray_GetNDArrayCFeatureVersion = 211, @@ -168,6 +174,7 @@ private: API_PyArray_DescrFromType = 45, API_PyArray_DescrFromScalar = 57, API_PyArray_FromAny = 69, + API_PyArray_Resize = 80, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, API_PyArray_DescrNewFromType = 9, @@ -197,6 +204,7 @@ private: DECL_NPY_API(PyArray_DescrFromType); DECL_NPY_API(PyArray_DescrFromScalar); DECL_NPY_API(PyArray_FromAny); + DECL_NPY_API(PyArray_Resize); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); DECL_NPY_API(PyArray_DescrNewFromType); @@ -652,6 +660,21 @@ public: return reinterpret_steal(api.PyArray_Squeeze_(m_ptr)); } + /// Resize array to given shape + /// If refcheck is true and more that one reference exist to this array + /// then resize will succeed only if it makes a reshape, i.e. original size doesn't change + void resize(ShapeContainer new_shape, bool refcheck = true) { + detail::npy_api::PyArray_Dims d = { + new_shape->data(), int(new_shape->size()) + }; + // try to resize, set ordering param to -1 cause it's not used anyway + object new_array = reinterpret_steal( + detail::npy_api::get().PyArray_Resize_(m_ptr, &d, int(refcheck), -1) + ); + if (!new_array) throw error_already_set(); + if (isinstance(new_array)) { *this = std::move(new_array); } + } + /// Ensure that the argument is a NumPy array /// In case of an error, nullptr is returned and the Python error is cleared. static array ensure(handle h, int ExtraFlags = 0) { -- cgit v1.2.3 From d400f60c96312f32fafcb15168bbf56970ff3d3a Mon Sep 17 00:00:00 2001 From: Cris Luengo Date: Wed, 5 Apr 2017 16:13:04 -0600 Subject: Python buffer objects can have negative strides. --- include/pybind11/numpy.h | 50 +++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 24 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index ba9402b..f86df9c 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -265,14 +265,14 @@ protected: const unsigned char *data_; // Storing the shape & strides in local variables (i.e. these arrays) allows the compiler to // make large performance gains on big, nested loops, but requires compile-time dimensions - conditional_t> - shape_, strides_; + conditional_t> shape_; + conditional_t> strides_; const size_t dims_; friend class pybind11::array; // Constructor for compile-time dimensions: template - unchecked_reference(const void *data, const size_t *shape, const size_t *strides, enable_if_t) + unchecked_reference(const void *data, const size_t *shape, const ssize_t *strides, enable_if_t) : data_{reinterpret_cast(data)}, dims_{Dims} { for (size_t i = 0; i < dims_; i++) { shape_[i] = shape[i]; @@ -281,7 +281,7 @@ protected: } // Constructor for runtime dimensions: template - unchecked_reference(const void *data, const size_t *shape, const size_t *strides, enable_if_t dims) + unchecked_reference(const void *data, const size_t *shape, const ssize_t *strides, enable_if_t dims) : data_{reinterpret_cast(data)}, shape_{shape}, strides_{strides}, dims_{dims} {} public: @@ -573,12 +573,12 @@ public: } /// Strides of the array - const size_t* strides() const { - return reinterpret_cast(detail::array_proxy(m_ptr)->strides); + const ssize_t* strides() const { + return reinterpret_cast(detail::array_proxy(m_ptr)->strides); } /// Stride along a given axis - size_t strides(size_t dim) const { + ssize_t strides(size_t dim) const { if (dim >= ndim()) fail_dim_check(dim, "invalid axis"); return strides()[dim]; @@ -702,9 +702,9 @@ protected: throw std::domain_error("array is not writeable"); } - static std::vector default_strides(const std::vector& shape, size_t itemsize) { + static std::vector default_strides(const std::vector& shape, size_t itemsize) { auto ndim = shape.size(); - std::vector strides(ndim); + std::vector strides(ndim); if (ndim) { std::fill(strides.begin(), strides.end(), itemsize); for (size_t i = 0; i < ndim - 1; i++) @@ -1133,7 +1133,7 @@ array_iterator array_end(const buffer_info& buffer) { class common_iterator { public: - using container_type = std::vector; + using container_type = std::vector; using value_type = container_type::value_type; using size_type = container_type::size_type; @@ -1175,7 +1175,7 @@ public: for (size_t i = 0; i < shape.size(); ++i) m_shape[i] = static_cast(shape[i]); - container_type strides(shape.size()); + std::vector strides(shape.size()); for (size_t i = 0; i < N; ++i) init_common_iterator(buffers[i], shape, m_common_iterator[i], strides); } @@ -1203,7 +1203,7 @@ private: void init_common_iterator(const buffer_info &buffer, const std::vector &shape, - common_iter &iterator, container_type &strides) { + common_iter &iterator, std::vector &strides) { auto buffer_shape_iter = buffer.shape.rbegin(); auto buffer_strides_iter = buffer.strides.rbegin(); auto shape_iter = shape.rbegin(); @@ -1211,7 +1211,7 @@ private: while (buffer_shape_iter != buffer.shape.rend()) { if (*shape_iter == *buffer_shape_iter) - *strides_iter = static_cast(*buffer_strides_iter); + *strides_iter = *buffer_strides_iter; else *strides_iter = 0; @@ -1283,10 +1283,11 @@ broadcast_trivial broadcast(const std::array &buffers, size_t &n // Check for C contiguity (but only if previous inputs were also C contiguous) if (trivial_broadcast_c) { - size_t expect_stride = buffers[i].itemsize; + ssize_t expect_stride = static_cast(buffers[i].itemsize); auto end = buffers[i].shape.crend(); - for (auto shape_iter = buffers[i].shape.crbegin(), stride_iter = buffers[i].strides.crbegin(); - trivial_broadcast_c && shape_iter != end; ++shape_iter, ++stride_iter) { + auto shape_iter = buffers[i].shape.crbegin(); + auto stride_iter = buffers[i].strides.crbegin(); + for (; trivial_broadcast_c && shape_iter != end; ++shape_iter, ++stride_iter) { if (expect_stride == *stride_iter) expect_stride *= *shape_iter; else @@ -1296,10 +1297,11 @@ broadcast_trivial broadcast(const std::array &buffers, size_t &n // Check for Fortran contiguity (if previous inputs were also F contiguous) if (trivial_broadcast_f) { - size_t expect_stride = buffers[i].itemsize; + ssize_t expect_stride = static_cast(buffers[i].itemsize); auto end = buffers[i].shape.cend(); - for (auto shape_iter = buffers[i].shape.cbegin(), stride_iter = buffers[i].strides.cbegin(); - trivial_broadcast_f && shape_iter != end; ++shape_iter, ++stride_iter) { + auto shape_iter = buffers[i].shape.cbegin(); + auto stride_iter = buffers[i].strides.cbegin(); + for (; trivial_broadcast_f && shape_iter != end; ++shape_iter, ++stride_iter) { if (expect_stride == *stride_iter) expect_stride *= *shape_iter; else @@ -1336,20 +1338,20 @@ struct vectorize_helper { auto trivial = broadcast(buffers, ndim, shape); size_t size = 1; - std::vector strides(ndim); + std::vector strides(ndim); if (ndim > 0) { if (trivial == broadcast_trivial::f_trivial) { - strides[0] = sizeof(Return); + strides[0] = static_cast(sizeof(Return)); for (size_t i = 1; i < ndim; ++i) { - strides[i] = strides[i - 1] * shape[i - 1]; + strides[i] = strides[i - 1] * static_cast(shape[i - 1]); size *= shape[i - 1]; } size *= shape[ndim - 1]; } else { - strides[ndim-1] = sizeof(Return); + strides[ndim-1] = static_cast(sizeof(Return)); for (size_t i = ndim - 1; i > 0; --i) { - strides[i - 1] = strides[i] * shape[i]; + strides[i - 1] = strides[i] * static_cast(shape[i]); size *= shape[i]; } size *= shape[0]; -- cgit v1.2.3 From 627da3f135d3e01396f4ae911dc15145c1856cf0 Mon Sep 17 00:00:00 2001 From: Cris Luengo Date: Thu, 6 Apr 2017 11:34:39 -0600 Subject: Making a copy when casting a numpy array with negative strides to Eigen. `EigenConformable::stride_compatible` returns false if the strides are negative. In this case, do not use `EigenConformable::stride`, as it is {0,0}. We cannot write negative strides in this element, as Eigen will throw an assertion if we do. The `type_caster` specialization for regular, dense Eigen matrices now does a second `array_t::ensure` to copy data in case of negative strides. I'm not sure that this is the best way to implement this. I have added "TODO" tags linking these changes to Eigen bug #747, which, when fixed, will allow Eigen to accept negative strides. --- include/pybind11/numpy.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index f86df9c..c1205fe 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -248,10 +248,10 @@ template using is_pod_struct = all_of< satisfies_none_of >; -template size_t byte_offset_unsafe(const Strides &) { return 0; } +template ssize_t byte_offset_unsafe(const Strides &) { return 0; } template -size_t byte_offset_unsafe(const Strides &strides, size_t i, Ix... index) { - return i * strides[Dim] + byte_offset_unsafe(strides, index...); +ssize_t byte_offset_unsafe(const Strides &strides, size_t i, Ix... index) { + return static_cast(i) * strides[Dim] + byte_offset_unsafe(strides, index...); } /** Proxy class providing unsafe, unchecked const access to array data. This is constructed through @@ -615,18 +615,18 @@ public: /// Byte offset from beginning of the array to a given index (full or partial). /// May throw if the index would lead to out of bounds access. - template size_t offset_at(Ix... index) const { + template ssize_t offset_at(Ix... index) const { if (sizeof...(index) > ndim()) fail_dim_check(sizeof...(index), "too many indices for an array"); return byte_offset(size_t(index)...); } - size_t offset_at() const { return 0; } + ssize_t offset_at() const { return 0; } /// Item count from beginning of the array to a given index (full or partial). /// May throw if the index would lead to out of bounds access. - template size_t index_at(Ix... index) const { - return offset_at(index...) / itemsize(); + template ssize_t index_at(Ix... index) const { + return offset_at(index...) / static_cast(itemsize()); } /** Returns a proxy object that provides access to the array's data without bounds or @@ -692,7 +692,7 @@ protected: " (ndim = " + std::to_string(ndim()) + ")"); } - template size_t byte_offset(Ix... index) const { + template ssize_t byte_offset(Ix... index) const { check_dimensions(index...); return detail::byte_offset_unsafe(strides(), size_t(index)...); } @@ -773,8 +773,8 @@ public: return sizeof(T); } - template size_t index_at(Ix... index) const { - return offset_at(index...) / itemsize(); + template ssize_t index_at(Ix... index) const { + return offset_at(index...) / static_cast(itemsize()); } template const T* data(Ix... index) const { @@ -789,14 +789,14 @@ public: template const T& at(Ix... index) const { if (sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); - return *(static_cast(array::data()) + byte_offset(size_t(index)...) / itemsize()); + return *(static_cast(array::data()) + byte_offset(size_t(index)...) / static_cast(itemsize())); } // Mutable reference to element at a given index template T& mutable_at(Ix... index) { if (sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); - return *(static_cast(array::mutable_data()) + byte_offset(size_t(index)...) / itemsize()); + return *(static_cast(array::mutable_data()) + byte_offset(size_t(index)...) / static_cast(itemsize())); } /** Returns a proxy object that provides access to the array's data without bounds or -- cgit v1.2.3 From b68959e822a619d1ad140f52c7197a29f4e6a06a Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 6 Apr 2017 18:16:35 -0400 Subject: Use numpy rather than Eigen for copying We're current copy by creating an Eigen::Map into the input numpy array, then assigning that to the basic eigen type, effectively having Eigen do the copy. That doesn't work for negative strides, though: Eigen doesn't allow them. This commit makes numpy do the copying instead by allocating the eigen type, then having numpy copy from the input array into a numpy reference into the eigen object's data. This also saves a copy when type conversion is required: numpy can do the conversion on-the-fly as part of the copy. Finally this commit also makes non-reference parameters respect the convert flag, declining the load when called in a noconvert pass with a convertible, but non-array input or an array with the wrong dtype. --- include/pybind11/numpy.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index c1205fe..0107366 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -152,6 +152,7 @@ struct npy_api { (PyTypeObject *, PyObject *, int, Py_intptr_t *, Py_intptr_t *, void *, int, PyObject *); PyObject *(*PyArray_DescrNewFromType_)(int); + int (*PyArray_CopyInto_)(PyObject *, PyObject *); PyObject *(*PyArray_NewCopy_)(PyObject *, int); PyTypeObject *PyArray_Type_; PyTypeObject *PyVoidArrType_Type_; @@ -175,6 +176,7 @@ private: API_PyArray_DescrFromScalar = 57, API_PyArray_FromAny = 69, API_PyArray_Resize = 80, + API_PyArray_CopyInto = 82, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, API_PyArray_DescrNewFromType = 9, @@ -205,6 +207,7 @@ private: DECL_NPY_API(PyArray_DescrFromScalar); DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_Resize); + DECL_NPY_API(PyArray_CopyInto); DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); DECL_NPY_API(PyArray_DescrNewFromType); -- cgit v1.2.3 From 30d43c4992f90ac7592fdbedd56c332717278c91 Mon Sep 17 00:00:00 2001 From: Cris Luengo Date: Fri, 14 Apr 2017 14:33:44 -0600 Subject: Now `shape`, `size`, `ndims` and `itemsize` are also signed integers. --- include/pybind11/numpy.h | 180 +++++++++++++++++++++++------------------------ 1 file changed, 90 insertions(+), 90 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0107366..72bb350 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -251,10 +251,10 @@ template using is_pod_struct = all_of< satisfies_none_of >; -template ssize_t byte_offset_unsafe(const Strides &) { return 0; } -template -ssize_t byte_offset_unsafe(const Strides &strides, size_t i, Ix... index) { - return static_cast(i) * strides[Dim] + byte_offset_unsafe(strides, index...); +template ssize_t byte_offset_unsafe(const Strides &) { return 0; } +template +ssize_t byte_offset_unsafe(const Strides &strides, ssize_t i, Ix... index) { + return i * strides[Dim] + byte_offset_unsafe(strides, index...); } /** Proxy class providing unsafe, unchecked const access to array data. This is constructed through @@ -268,23 +268,23 @@ protected: const unsigned char *data_; // Storing the shape & strides in local variables (i.e. these arrays) allows the compiler to // make large performance gains on big, nested loops, but requires compile-time dimensions - conditional_t> shape_; - conditional_t> strides_; - const size_t dims_; + conditional_t> + shape_, strides_; + const ssize_t dims_; friend class pybind11::array; // Constructor for compile-time dimensions: template - unchecked_reference(const void *data, const size_t *shape, const ssize_t *strides, enable_if_t) + unchecked_reference(const void *data, const ssize_t *shape, const ssize_t *strides, enable_if_t) : data_{reinterpret_cast(data)}, dims_{Dims} { - for (size_t i = 0; i < dims_; i++) { + for (size_t i = 0; i < (size_t) dims_; i++) { shape_[i] = shape[i]; strides_[i] = strides[i]; } } // Constructor for runtime dimensions: template - unchecked_reference(const void *data, const size_t *shape, const ssize_t *strides, enable_if_t dims) + unchecked_reference(const void *data, const ssize_t *shape, const ssize_t *strides, enable_if_t dims) : data_{reinterpret_cast(data)}, shape_{shape}, strides_{strides}, dims_{dims} {} public: @@ -295,39 +295,39 @@ public: template const T &operator()(Ix... index) const { static_assert(sizeof...(Ix) == Dims || Dynamic, "Invalid number of indices for unchecked array reference"); - return *reinterpret_cast(data_ + byte_offset_unsafe(strides_, size_t(index)...)); + return *reinterpret_cast(data_ + byte_offset_unsafe(strides_, ssize_t(index)...)); } /** Unchecked const reference access to data; this operator only participates if the reference * is to a 1-dimensional array. When present, this is exactly equivalent to `obj(index)`. */ - template > - const T &operator[](size_t index) const { return operator()(index); } + template > + const T &operator[](ssize_t index) const { return operator()(index); } /// Pointer access to the data at the given indices. - template const T *data(Ix... ix) const { return &operator()(size_t(ix)...); } + template const T *data(Ix... ix) const { return &operator()(ssize_t(ix)...); } /// Returns the item size, i.e. sizeof(T) - constexpr static size_t itemsize() { return sizeof(T); } + constexpr static ssize_t itemsize() { return sizeof(T); } /// Returns the shape (i.e. size) of dimension `dim` - size_t shape(size_t dim) const { return shape_[dim]; } + ssize_t shape(ssize_t dim) const { return shape_[(size_t) dim]; } /// Returns the number of dimensions of the array - size_t ndim() const { return dims_; } + ssize_t ndim() const { return dims_; } /// Returns the total number of elements in the referenced array, i.e. the product of the shapes template - enable_if_t size() const { - return std::accumulate(shape_.begin(), shape_.end(), (size_t) 1, std::multiplies()); + enable_if_t size() const { + return std::accumulate(shape_.begin(), shape_.end(), (ssize_t) 1, std::multiplies()); } template - enable_if_t size() const { - return std::accumulate(shape_, shape_ + ndim(), (size_t) 1, std::multiplies()); + enable_if_t size() const { + return std::accumulate(shape_, shape_ + ndim(), (ssize_t) 1, std::multiplies()); } /// Returns the total number of bytes used by the referenced data. Note that the actual span in /// memory may be larger if the referenced array has non-contiguous strides (e.g. for a slice). - size_t nbytes() const { + ssize_t nbytes() const { return size() * itemsize(); } }; @@ -349,11 +349,11 @@ public: * reference is to a 1-dimensional array (or has runtime dimensions). When present, this is * exactly equivalent to `obj(index)`. */ - template > - T &operator[](size_t index) { return operator()(index); } + template > + T &operator[](ssize_t index) { return operator()(index); } /// Mutable pointer access to the data at the given indices. - template T *mutable_data(Ix... ix) { return &operator()(size_t(ix)...); } + template T *mutable_data(Ix... ix) { return &operator()(ssize_t(ix)...); } }; template @@ -381,7 +381,7 @@ public: dtype(const char *format) : dtype(std::string(format)) { } - dtype(list names, list formats, list offsets, size_t itemsize) { + dtype(list names, list formats, list offsets, ssize_t itemsize) { dict args; args["names"] = names; args["formats"] = formats; @@ -404,8 +404,8 @@ public: } /// Size of the data type in bytes. - size_t itemsize() const { - return (size_t) detail::array_descriptor_proxy(m_ptr)->elsize; + ssize_t itemsize() const { + return detail::array_descriptor_proxy(m_ptr)->elsize; } /// Returns true for structured data types. @@ -425,7 +425,7 @@ private: return reinterpret_borrow(obj); } - dtype strip_padding(size_t itemsize) { + dtype strip_padding(ssize_t itemsize) { // Recursively strip all void fields with empty names that are generated for // padding fields (as of NumPy v1.11). if (!has_fields()) @@ -501,7 +501,7 @@ public: api.PyArray_Type_, descr.release().ptr(), (int) ndim, shape->data(), strides->data(), const_cast(ptr), flags, nullptr)); if (!tmp) - pybind11_fail("NumPy: unable to create array!"); + throw error_already_set(); if (ptr) { if (base) { api.PyArray_SetBaseObject_(tmp.ptr(), base.inc_ref().ptr()); @@ -528,7 +528,7 @@ public: : array(std::move(shape), {}, ptr, base) { } template - explicit array(size_t count, const T *ptr, handle base = handle()) : array({count}, {}, ptr, base) { } + explicit array(ssize_t count, const T *ptr, handle base = handle()) : array({count}, {}, ptr, base) { } explicit array(const buffer_info &info) : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } @@ -539,23 +539,23 @@ public: } /// Total number of elements - size_t size() const { - return std::accumulate(shape(), shape() + ndim(), (size_t) 1, std::multiplies()); + ssize_t size() const { + return std::accumulate(shape(), shape() + ndim(), (ssize_t) 1, std::multiplies()); } /// Byte size of a single element - size_t itemsize() const { - return (size_t) detail::array_descriptor_proxy(detail::array_proxy(m_ptr)->descr)->elsize; + ssize_t itemsize() const { + return detail::array_descriptor_proxy(detail::array_proxy(m_ptr)->descr)->elsize; } /// Total number of bytes - size_t nbytes() const { + ssize_t nbytes() const { return size() * itemsize(); } /// Number of dimensions - size_t ndim() const { - return (size_t) detail::array_proxy(m_ptr)->nd; + ssize_t ndim() const { + return detail::array_proxy(m_ptr)->nd; } /// Base object @@ -564,12 +564,12 @@ public: } /// Dimensions of the array - const size_t* shape() const { - return reinterpret_cast(detail::array_proxy(m_ptr)->dimensions); + const ssize_t* shape() const { + return detail::array_proxy(m_ptr)->dimensions; } /// Dimension along a given axis - size_t shape(size_t dim) const { + ssize_t shape(ssize_t dim) const { if (dim >= ndim()) fail_dim_check(dim, "invalid axis"); return shape()[dim]; @@ -577,11 +577,11 @@ public: /// Strides of the array const ssize_t* strides() const { - return reinterpret_cast(detail::array_proxy(m_ptr)->strides); + return detail::array_proxy(m_ptr)->strides; } /// Stride along a given axis - ssize_t strides(size_t dim) const { + ssize_t strides(ssize_t dim) const { if (dim >= ndim()) fail_dim_check(dim, "invalid axis"); return strides()[dim]; @@ -619,9 +619,9 @@ public: /// Byte offset from beginning of the array to a given index (full or partial). /// May throw if the index would lead to out of bounds access. template ssize_t offset_at(Ix... index) const { - if (sizeof...(index) > ndim()) + if ((ssize_t) sizeof...(index) > ndim()) fail_dim_check(sizeof...(index), "too many indices for an array"); - return byte_offset(size_t(index)...); + return byte_offset(ssize_t(index)...); } ssize_t offset_at() const { return 0; } @@ -629,7 +629,7 @@ public: /// Item count from beginning of the array to a given index (full or partial). /// May throw if the index would lead to out of bounds access. template ssize_t index_at(Ix... index) const { - return offset_at(index...) / static_cast(itemsize()); + return offset_at(index...) / itemsize(); } /** Returns a proxy object that provides access to the array's data without bounds or @@ -638,7 +638,7 @@ public: * and the caller must take care not to access invalid dimensions or dimension indices. */ template detail::unchecked_mutable_reference mutable_unchecked() { - if (Dims >= 0 && ndim() != (size_t) Dims) + if (Dims >= 0 && ndim() != Dims) throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); return detail::unchecked_mutable_reference(mutable_data(), shape(), strides(), ndim()); @@ -651,7 +651,7 @@ public: * invalid dimensions or dimension indices. */ template detail::unchecked_reference unchecked() const { - if (Dims >= 0 && ndim() != (size_t) Dims) + if (Dims >= 0 && ndim() != Dims) throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); return detail::unchecked_reference(data(), shape(), strides(), ndim()); @@ -690,14 +690,14 @@ public: protected: template friend struct detail::npy_format_descriptor; - void fail_dim_check(size_t dim, const std::string& msg) const { + void fail_dim_check(ssize_t dim, const std::string& msg) const { throw index_error(msg + ": " + std::to_string(dim) + " (ndim = " + std::to_string(ndim()) + ")"); } template ssize_t byte_offset(Ix... index) const { check_dimensions(index...); - return detail::byte_offset_unsafe(strides(), size_t(index)...); + return detail::byte_offset_unsafe(strides(), ssize_t(index)...); } void check_writeable() const { @@ -705,7 +705,7 @@ protected: throw std::domain_error("array is not writeable"); } - static std::vector default_strides(const std::vector& shape, size_t itemsize) { + static std::vector default_strides(const std::vector& shape, ssize_t itemsize) { auto ndim = shape.size(); std::vector strides(ndim); if (ndim) { @@ -718,12 +718,12 @@ protected: } template void check_dimensions(Ix... index) const { - check_dimensions_impl(size_t(0), shape(), size_t(index)...); + check_dimensions_impl(ssize_t(0), shape(), ssize_t(index)...); } - void check_dimensions_impl(size_t, const size_t*) const { } + void check_dimensions_impl(ssize_t, const ssize_t*) const { } - template void check_dimensions_impl(size_t axis, const size_t* shape, size_t i, Ix... index) const { + template void check_dimensions_impl(ssize_t axis, const ssize_t* shape, ssize_t i, Ix... index) const { if (i >= *shape) { throw index_error(std::string("index ") + std::to_string(i) + " is out of bounds for axis " + std::to_string(axis) + @@ -772,12 +772,12 @@ public: explicit array_t(size_t count, const T *ptr = nullptr, handle base = handle()) : array({count}, {}, ptr, base) { } - constexpr size_t itemsize() const { + constexpr ssize_t itemsize() const { return sizeof(T); } template ssize_t index_at(Ix... index) const { - return offset_at(index...) / static_cast(itemsize()); + return offset_at(index...) / itemsize(); } template const T* data(Ix... index) const { @@ -792,14 +792,14 @@ public: template const T& at(Ix... index) const { if (sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); - return *(static_cast(array::data()) + byte_offset(size_t(index)...) / static_cast(itemsize())); + return *(static_cast(array::data()) + byte_offset(ssize_t(index)...) / itemsize()); } // Mutable reference to element at a given index template T& mutable_at(Ix... index) { if (sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); - return *(static_cast(array::mutable_data()) + byte_offset(size_t(index)...) / static_cast(itemsize())); + return *(static_cast(array::mutable_data()) + byte_offset(ssize_t(index)...) / itemsize()); } /** Returns a proxy object that provides access to the array's data without bounds or @@ -949,16 +949,16 @@ public: struct field_descriptor { const char *name; - size_t offset; - size_t size; - size_t alignment; + ssize_t offset; + ssize_t size; + ssize_t alignment; std::string format; dtype descr; }; inline PYBIND11_NOINLINE void register_structured_dtype( const std::initializer_list& fields, - const std::type_info& tinfo, size_t itemsize, + const std::type_info& tinfo, ssize_t itemsize, bool (*direct_converter)(PyObject *, void *&)) { auto& numpy_internals = get_numpy_internals(); @@ -986,7 +986,7 @@ inline PYBIND11_NOINLINE void register_structured_dtype( std::vector ordered_fields(fields); std::sort(ordered_fields.begin(), ordered_fields.end(), [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); - size_t offset = 0; + ssize_t offset = 0; std::ostringstream oss; oss << "T{"; for (auto& field : ordered_fields) { @@ -1142,7 +1142,7 @@ public: common_iterator() : p_ptr(0), m_strides() {} - common_iterator(void* ptr, const container_type& strides, const std::vector& shape) + common_iterator(void* ptr, const container_type& strides, const container_type& shape) : p_ptr(reinterpret_cast(ptr)), m_strides(strides.size()) { m_strides.back() = static_cast(strides.back()); for (size_type i = m_strides.size() - 1; i != 0; --i) { @@ -1167,18 +1167,18 @@ private: template class multi_array_iterator { public: - using container_type = std::vector; + using container_type = std::vector; multi_array_iterator(const std::array &buffers, - const std::vector &shape) + const container_type &shape) : m_shape(shape.size()), m_index(shape.size(), 0), m_common_iterator() { // Manual copy to avoid conversion warning if using std::copy for (size_t i = 0; i < shape.size(); ++i) - m_shape[i] = static_cast(shape[i]); + m_shape[i] = shape[i]; - std::vector strides(shape.size()); + container_type strides(shape.size()); for (size_t i = 0; i < N; ++i) init_common_iterator(buffers[i], shape, m_common_iterator[i], strides); } @@ -1205,8 +1205,9 @@ private: using common_iter = common_iterator; void init_common_iterator(const buffer_info &buffer, - const std::vector &shape, - common_iter &iterator, std::vector &strides) { + const container_type &shape, + common_iter &iterator, + container_type &strides) { auto buffer_shape_iter = buffer.shape.rbegin(); auto buffer_strides_iter = buffer.strides.rbegin(); auto shape_iter = shape.rbegin(); @@ -1245,13 +1246,13 @@ enum class broadcast_trivial { non_trivial, c_trivial, f_trivial }; // singleton or a full-size, C-contiguous (`c_trivial`) or Fortran-contiguous (`f_trivial`) storage // buffer; returns `non_trivial` otherwise. template -broadcast_trivial broadcast(const std::array &buffers, size_t &ndim, std::vector &shape) { - ndim = std::accumulate(buffers.begin(), buffers.end(), size_t(0), [](size_t res, const buffer_info& buf) { +broadcast_trivial broadcast(const std::array &buffers, ssize_t &ndim, std::vector &shape) { + ndim = std::accumulate(buffers.begin(), buffers.end(), ssize_t(0), [](ssize_t res, const buffer_info& buf) { return std::max(res, buf.ndim); }); shape.clear(); - shape.resize(ndim, 1); + shape.resize((size_t) ndim, 1); // Figure out the output size, and make sure all input arrays conform (i.e. are either size 1 or // the full size). @@ -1286,11 +1287,10 @@ broadcast_trivial broadcast(const std::array &buffers, size_t &n // Check for C contiguity (but only if previous inputs were also C contiguous) if (trivial_broadcast_c) { - ssize_t expect_stride = static_cast(buffers[i].itemsize); + ssize_t expect_stride = buffers[i].itemsize; auto end = buffers[i].shape.crend(); - auto shape_iter = buffers[i].shape.crbegin(); - auto stride_iter = buffers[i].strides.crbegin(); - for (; trivial_broadcast_c && shape_iter != end; ++shape_iter, ++stride_iter) { + for (auto shape_iter = buffers[i].shape.crbegin(), stride_iter = buffers[i].strides.crbegin(); + trivial_broadcast_c && shape_iter != end; ++shape_iter, ++stride_iter) { if (expect_stride == *stride_iter) expect_stride *= *shape_iter; else @@ -1300,11 +1300,10 @@ broadcast_trivial broadcast(const std::array &buffers, size_t &n // Check for Fortran contiguity (if previous inputs were also F contiguous) if (trivial_broadcast_f) { - ssize_t expect_stride = static_cast(buffers[i].itemsize); + ssize_t expect_stride = buffers[i].itemsize; auto end = buffers[i].shape.cend(); - auto shape_iter = buffers[i].shape.cbegin(); - auto stride_iter = buffers[i].strides.cbegin(); - for (; trivial_broadcast_f && shape_iter != end; ++shape_iter, ++stride_iter) { + for (auto shape_iter = buffers[i].shape.cbegin(), stride_iter = buffers[i].strides.cbegin(); + trivial_broadcast_f && shape_iter != end; ++shape_iter, ++stride_iter) { if (expect_stride == *stride_iter) expect_stride *= *shape_iter; else @@ -1336,25 +1335,26 @@ struct vectorize_helper { std::array buffers {{ args.request()... }}; /* Determine dimensions parameters of output array */ - size_t ndim = 0; - std::vector shape(0); - auto trivial = broadcast(buffers, ndim, shape); + ssize_t nd = 0; + std::vector shape(0); + auto trivial = broadcast(buffers, nd, shape); + size_t ndim = (size_t) nd; - size_t size = 1; + ssize_t size = 1; std::vector strides(ndim); if (ndim > 0) { if (trivial == broadcast_trivial::f_trivial) { - strides[0] = static_cast(sizeof(Return)); + strides[0] = sizeof(Return); for (size_t i = 1; i < ndim; ++i) { - strides[i] = strides[i - 1] * static_cast(shape[i - 1]); + strides[i] = strides[i - 1] * shape[i - 1]; size *= shape[i - 1]; } size *= shape[ndim - 1]; } else { - strides[ndim-1] = static_cast(sizeof(Return)); + strides[ndim-1] = sizeof(Return); for (size_t i = ndim - 1; i > 0; --i) { - strides[i - 1] = strides[i] * static_cast(shape[i]); + strides[i - 1] = strides[i] * shape[i]; size *= shape[i]; } size *= shape[0]; @@ -1372,7 +1372,7 @@ struct vectorize_helper { if (trivial == broadcast_trivial::non_trivial) { apply_broadcast(buffers, buf, index); } else { - for (size_t i = 0; i < size; ++i) + for (ssize_t i = 0; i < size; ++i) output[i] = f((reinterpret_cast(buffers[Index].ptr)[buffers[Index].size == 1 ? 0 : i])...); } -- cgit v1.2.3 From 8e0d832c7d84a3dd77f35828e1ff35045fc36fe7 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 10 May 2017 10:21:01 +0200 Subject: Support arrays inside PYBIND11_NUMPY_DTYPE (#832) Resolves #800. Both C++ arrays and std::array are supported, including mixtures like std::array[4]. In a multi-dimensional array of char, the last dimension is used to construct a numpy string type. --- include/pybind11/numpy.h | 65 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 72bb350..62901b8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -246,6 +246,46 @@ template struct is_std_array> : std::tru template struct is_complex : std::false_type { }; template struct is_complex> : std::true_type { }; +template struct array_info_scalar { + typedef T type; + static constexpr bool is_array = false; + static constexpr bool is_empty = false; + static PYBIND11_DESCR extents() { return _(""); } + static void append_extents(list& /* shape */) { } +}; +// Computes underlying type and a comma-separated list of extents for array +// types (any mix of std::array and built-in arrays). An array of char is +// treated as scalar because it gets special handling. +template struct array_info : array_info_scalar { }; +template struct array_info> { + using type = typename array_info::type; + static constexpr bool is_array = true; + static constexpr bool is_empty = (N == 0) || array_info::is_empty; + static constexpr size_t extent = N; + + // appends the extents to shape + static void append_extents(list& shape) { + shape.append(N); + array_info::append_extents(shape); + } + + template::is_array, int> = 0> + static PYBIND11_DESCR extents() { + return _(); + } + + template::is_array, int> = 0> + static PYBIND11_DESCR extents() { + return concat(_(), array_info::extents()); + } +}; +// For numpy we have special handling for arrays of characters, so we don't include +// the size in the array extents. +template struct array_info : array_info_scalar { }; +template struct array_info> : array_info_scalar> { }; +template struct array_info : array_info> { }; +template using remove_all_extents_t = typename array_info::type; + template using is_pod_struct = all_of< std::is_pod, // since we're accessing directly in memory we need a POD type satisfies_none_of @@ -745,6 +785,8 @@ protected: template class array_t : public array { public: + static_assert(!detail::array_info::is_array, "Array types cannot be used with array_t"); + using value_type = T; array_t() : array(0, static_cast(nullptr)) {} @@ -871,6 +913,15 @@ struct format_descriptor::value>> { } }; +template +struct format_descriptor::is_array>> { + static std::string format() { + using detail::_; + PYBIND11_DESCR extents = _("(") + detail::array_info::extents() + _(")"); + return extents.text() + format_descriptor>::format(); + } +}; + NAMESPACE_BEGIN(detail) template struct pyobject_caster> { @@ -939,6 +990,20 @@ template struct npy_format_descriptor { PYBIND11_DECL_CHAR_F template struct npy_format_descriptor> { PYBIND11_DECL_CHAR_FMT }; #undef PYBIND11_DECL_CHAR_FMT +template struct npy_format_descriptor::is_array>> { +private: + using base_descr = npy_format_descriptor::type>; +public: + static_assert(!array_info::is_empty, "Zero-sized arrays are not supported"); + + static PYBIND11_DESCR name() { return _("(") + array_info::extents() + _(")") + base_descr::name(); } + static pybind11::dtype dtype() { + list shape; + array_info::append_extents(shape); + return pybind11::dtype::from_args(pybind11::make_tuple(base_descr::dtype(), shape)); + } +}; + template struct npy_format_descriptor::value>> { private: using base_descr = npy_format_descriptor::type>; -- cgit v1.2.3 From b82c0f0a2daa5a672afd130c56989731b8d9dd29 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 10 May 2017 11:36:24 +0200 Subject: Allow std::complex field with PYBIND11_NUMPY_DTYPE (#831) This exposed a few underlying issues: 1. is_pod_struct was too strict to allow this. I've relaxed it to require only trivially copyable and standard layout, rather than POD (which additionally requires a trivial constructor, which std::complex violates). 2. format_descriptor>::format() returned numpy format strings instead of PEP3118 format strings, but register_dtype feeds format codes of its fields to _dtype_from_pep3118. I've changed it to return PEP3118 format codes. format_descriptor is a public type, so this may be considered an incompatible change. 3. register_structured_dtype tried to be smart about whether to mark fields as unaligned (with ^). However, it's examining the C++ alignment, rather than what numpy (or possibly PEP3118) thinks the alignment should be. For complex values those are different. I've made it mark all fields as ^ unconditionally, which should always be safe even if they are aligned, because we explicitly mark the padding. --- include/pybind11/numpy.h | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 62901b8..72840c4 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -287,7 +287,14 @@ template struct array_info : array_info using remove_all_extents_t = typename array_info::type; template using is_pod_struct = all_of< - std::is_pod, // since we're accessing directly in memory we need a POD type + std::is_standard_layout, // since we're accessing directly in memory we need a standard layout type +#if !defined(__GNUG__) || defined(__clang__) || __GNUC__ >= 5 + std::is_trivially_copyable, +#else + // GCC 4 doesn't implement is_trivially_copyable, so approximate it + std::is_trivially_destructible, + satisfies_any_of, +#endif satisfies_none_of >; @@ -1016,7 +1023,6 @@ struct field_descriptor { const char *name; ssize_t offset; ssize_t size; - ssize_t alignment; std::string format; dtype descr; }; @@ -1053,13 +1059,15 @@ inline PYBIND11_NOINLINE void register_structured_dtype( [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); ssize_t offset = 0; std::ostringstream oss; - oss << "T{"; + // mark the structure as unaligned with '^', because numpy and C++ don't + // always agree about alignment (particularly for complex), and we're + // explicitly listing all our padding. This depends on none of the fields + // overriding the endianness. Putting the ^ in front of individual fields + // isn't guaranteed to work due to https://github.com/numpy/numpy/issues/9049 + oss << "^T{"; for (auto& field : ordered_fields) { if (field.offset > offset) oss << (field.offset - offset) << 'x'; - // mark unaligned fields with '^' (unaligned native type) - if (field.offset % field.alignment) - oss << '^'; oss << field.format << ':' << field.name << ':'; offset = field.offset + field.size; } @@ -1121,7 +1129,6 @@ private: #define PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, Name) \ ::pybind11::detail::field_descriptor { \ Name, offsetof(T, Field), sizeof(decltype(std::declval().Field)), \ - alignof(decltype(std::declval().Field)), \ ::pybind11::format_descriptor().Field)>::format(), \ ::pybind11::detail::npy_format_descriptor().Field)>::dtype() \ } -- cgit v1.2.3 From d2da33a34a255c77e96f1add10a2ef05b0240f39 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sat, 20 May 2017 11:21:19 -0400 Subject: static_assert should be testing ssize_t not size_t The numpy strides/sizes/etc. are signed now, but the static_assert didn't get updated to match. --- include/pybind11/numpy.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 72840c4..6e8b1be 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -29,10 +29,10 @@ #endif /* This will be true on all flat address space platforms and allows us to reduce the - whole npy_intp / size_t / Py_intptr_t business down to just size_t for all size + whole npy_intp / ssize_t / Py_intptr_t business down to just ssize_t for all size and dimension types (e.g. shape, strides, indexing), instead of inflicting this upon the library user. */ -static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); +static_assert(sizeof(ssize_t) == sizeof(Py_intptr_t), "ssize_t != Py_intptr_t"); NAMESPACE_BEGIN(pybind11) @@ -518,8 +518,8 @@ public: array() : array({{0}}, static_cast(nullptr)) {} - using ShapeContainer = detail::any_container; - using StridesContainer = detail::any_container; + using ShapeContainer = detail::any_container; + using StridesContainer = detail::any_container; // Constructs an array taking shape/strides from arbitrary container types array(const pybind11::dtype &dt, ShapeContainer shape, StridesContainer strides, @@ -752,7 +752,7 @@ protected: throw std::domain_error("array is not writeable"); } - static std::vector default_strides(const std::vector& shape, ssize_t itemsize) { + static std::vector default_strides(const std::vector& shape, ssize_t itemsize) { auto ndim = shape.size(); std::vector strides(ndim); if (ndim) { -- cgit v1.2.3 From 37b2383a646c93f531ae4275c0957902b18afe03 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 22 May 2017 12:06:16 -0400 Subject: Style cleanup of javadoc-style comments This changes javadoc-style documenting comments from: /** Text starts here * and continues here */ to: /** * Test starts here * and continues here */ which looks a little better, and also matches the javadoc-recommended way of writing documenting comments. --- include/pybind11/numpy.h | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6e8b1be..8901d97 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -304,7 +304,8 @@ ssize_t byte_offset_unsafe(const Strides &strides, ssize_t i, Ix... index) { return i * strides[Dim] + byte_offset_unsafe(strides, index...); } -/** Proxy class providing unsafe, unchecked const access to array data. This is constructed through +/** + * Proxy class providing unsafe, unchecked const access to array data. This is constructed through * the `unchecked()` method of `array` or the `unchecked()` method of `array_t`. `Dims` * will be -1 for dimensions determined at runtime. */ @@ -335,7 +336,8 @@ protected: : data_{reinterpret_cast(data)}, shape_{shape}, strides_{strides}, dims_{dims} {} public: - /** Unchecked const reference access to data at the given indices. For a compile-time known + /** + * Unchecked const reference access to data at the given indices. For a compile-time known * number of dimensions, this requires the correct number of arguments; for run-time * dimensionality, this is not checked (and so is up to the caller to use safely). */ @@ -344,7 +346,8 @@ public: "Invalid number of indices for unchecked array reference"); return *reinterpret_cast(data_ + byte_offset_unsafe(strides_, ssize_t(index)...)); } - /** Unchecked const reference access to data; this operator only participates if the reference + /** + * Unchecked const reference access to data; this operator only participates if the reference * is to a 1-dimensional array. When present, this is exactly equivalent to `obj(index)`. */ template > @@ -392,7 +395,8 @@ public: "Invalid number of indices for unchecked array reference"); return const_cast(ConstBase::operator()(index...)); } - /** Mutable, unchecked access data at the given index; this operator only participates if the + /** + * Mutable, unchecked access data at the given index; this operator only participates if the * reference is to a 1-dimensional array (or has runtime dimensions). When present, this is * exactly equivalent to `obj(index)`. */ @@ -679,7 +683,8 @@ public: return offset_at(index...) / itemsize(); } - /** Returns a proxy object that provides access to the array's data without bounds or + /** + * Returns a proxy object that provides access to the array's data without bounds or * dimensionality checking. Will throw if the array is missing the `writeable` flag. Use with * care: the array must not be destroyed or reshaped for the duration of the returned object, * and the caller must take care not to access invalid dimensions or dimension indices. @@ -691,7 +696,8 @@ public: return detail::unchecked_mutable_reference(mutable_data(), shape(), strides(), ndim()); } - /** Returns a proxy object that provides const access to the array's data without bounds or + /** + * Returns a proxy object that provides const access to the array's data without bounds or * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the * underlying array have the `writable` flag. Use with care: the array must not be destroyed or * reshaped for the duration of the returned object, and the caller must take care not to access @@ -851,7 +857,8 @@ public: return *(static_cast(array::mutable_data()) + byte_offset(ssize_t(index)...) / itemsize()); } - /** Returns a proxy object that provides access to the array's data without bounds or + /** + * Returns a proxy object that provides access to the array's data without bounds or * dimensionality checking. Will throw if the array is missing the `writeable` flag. Use with * care: the array must not be destroyed or reshaped for the duration of the returned object, * and the caller must take care not to access invalid dimensions or dimension indices. @@ -860,7 +867,8 @@ public: return array::mutable_unchecked(); } - /** Returns a proxy object that provides const access to the array's data without bounds or + /** + * Returns a proxy object that provides const access to the array's data without bounds or * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped * for the duration of the returned object, and the caller must take care not to access invalid -- cgit v1.2.3 From 41f8da4a95fae187cb74cc5dc4acc6d525bdeeda Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 27 Mar 2017 11:03:16 -0300 Subject: array_t: make c_style/f_style work for array creation Currently if you construct an `array_t` with a shape but not strides you get a C-style array; the only way to get F-style strides was to calculate the strides manually. This commit fixes that by adding logic to use f_style strides when the flag is set. This also simplifies the existing c_style stride logic. --- include/pybind11/numpy.h | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8901d97..2395b1f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -530,7 +530,7 @@ public: const void *ptr = nullptr, handle base = handle()) { if (strides->empty()) - *strides = default_strides(*shape, dt.itemsize()); + *strides = c_strides(*shape, dt.itemsize()); auto ndim = shape->size(); if (ndim != strides->size()) @@ -758,15 +758,21 @@ protected: throw std::domain_error("array is not writeable"); } - static std::vector default_strides(const std::vector& shape, ssize_t itemsize) { + // Default, C-style strides + static std::vector c_strides(const std::vector &shape, ssize_t itemsize) { auto ndim = shape.size(); - std::vector strides(ndim); - if (ndim) { - std::fill(strides.begin(), strides.end(), itemsize); - for (size_t i = 0; i < ndim - 1; i++) - for (size_t j = 0; j < ndim - 1 - i; j++) - strides[j] *= shape[ndim - 1 - i]; - } + std::vector strides(ndim, itemsize); + for (size_t i = ndim - 1; i > 0; --i) + strides[i - 1] = strides[i] * shape[i]; + return strides; + } + + // F-style strides; default when constructing an array_t with `ExtraFlags & f_style` + static std::vector f_strides(const std::vector &shape, ssize_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim, itemsize); + for (size_t i = 1; i < ndim; ++i) + strides[i] = strides[i - 1] * shape[i - 1]; return strides; } @@ -797,6 +803,11 @@ protected: }; template class array_t : public array { +private: + struct private_ctor {}; + // Delegating constructor needed when both moving and accessing in the same constructor + array_t(private_ctor, ShapeContainer &&shape, StridesContainer &&strides, const T *ptr, handle base) + : array(std::move(shape), std::move(strides), ptr, base) {} public: static_assert(!detail::array_info::is_array, "Array types cannot be used with array_t"); @@ -822,7 +833,9 @@ public: : array(std::move(shape), std::move(strides), ptr, base) { } explicit array_t(ShapeContainer shape, const T *ptr = nullptr, handle base = handle()) - : array(std::move(shape), ptr, base) { } + : array_t(private_ctor{}, std::move(shape), + ExtraFlags & f_style ? f_strides(*shape, itemsize()) : c_strides(*shape, itemsize()), + ptr, base) { } explicit array_t(size_t count, const T *ptr = nullptr, handle base = handle()) : array({count}, {}, ptr, base) { } -- cgit v1.2.3 From f3ce00eaed443ae1375364866dc74c10bd864558 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 26 Mar 2017 00:51:40 -0300 Subject: vectorize: pass-through of non-vectorizable args This extends py::vectorize to automatically pass through non-vectorizable arguments. This removes the need for the documented "explicitly exclude an argument" workaround. Vectorization now applies to arithmetic, std::complex, and POD types, passed as plain value or by const lvalue reference (previously only pass-by-value types were supported). Non-const lvalue references and any other types are passed through as-is. Functions with rvalue reference arguments (whether vectorizable or not) are explicitly prohibited: an rvalue reference is inherently not something that can be passed multiple times and is thus unsuitable to being in a vectorized function. The vectorize returned value is also now more sensitive to inputs: previously it would return by value when all inputs are of size 1; this is now amended to having all inputs of size 1 *and* 0 dimensions. Thus if you pass in, for example, [[1]], you get back a 1x1, 2D array, while previously you got back just the resulting single value. Vectorization of member function specializations is now also supported via `py::vectorize(&Class::method)`; this required passthrough support for the initial object pointer on the wrapping function pointer. --- include/pybind11/numpy.h | 192 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 131 insertions(+), 61 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2395b1f..1e12ede 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1289,8 +1289,8 @@ public: return *this; } - template const T& data() const { - return *reinterpret_cast(m_common_iterator[K].data()); + template T* data() const { + return reinterpret_cast(m_common_iterator[K].data()); } private: @@ -1340,7 +1340,7 @@ enum class broadcast_trivial { non_trivial, c_trivial, f_trivial }; // buffer; returns `non_trivial` otherwise. template broadcast_trivial broadcast(const std::array &buffers, ssize_t &ndim, std::vector &shape) { - ndim = std::accumulate(buffers.begin(), buffers.end(), ssize_t(0), [](ssize_t res, const buffer_info& buf) { + ndim = std::accumulate(buffers.begin(), buffers.end(), ssize_t(0), [](ssize_t res, const buffer_info &buf) { return std::max(res, buf.ndim); }); @@ -1411,21 +1411,63 @@ broadcast_trivial broadcast(const std::array &buffers, ssize_t & broadcast_trivial::non_trivial; } +template +struct vectorize_arg { + static_assert(!std::is_rvalue_reference::value, "Functions with rvalue reference arguments cannot be vectorized"); + // The wrapped function gets called with this type: + using call_type = remove_reference_t; + // Is this a vectorized argument? + static constexpr bool vectorize = + satisfies_any_of::value && + satisfies_none_of::value && + (!std::is_reference::value || + (std::is_lvalue_reference::value && std::is_const::value)); + // Accept this type: an array for vectorized types, otherwise the type as-is: + using type = conditional_t, array::forcecast>, T>; +}; + template struct vectorize_helper { - typename std::remove_reference::type f; +private: static constexpr size_t N = sizeof...(Args); + static constexpr size_t NVectorized = constexpr_sum(vectorize_arg::vectorize...); + static_assert(NVectorized >= 1, + "pybind11::vectorize(...) requires a function with at least one vectorizable argument"); +public: template - explicit vectorize_helper(T&&f) : f(std::forward(f)) { } + explicit vectorize_helper(T &&f) : f(std::forward(f)) { } - object operator()(array_t... args) { - return run(args..., make_index_sequence()); + object operator()(typename vectorize_arg::type... args) { + return run(args..., + make_index_sequence(), + select_indices::vectorize...>(), + make_index_sequence()); } - template object run(array_t&... args, index_sequence index) { - /* Request buffers from all parameters */ - std::array buffers {{ args.request()... }}; +private: + remove_reference_t f; + + template using param_n_t = typename pack_element::call_type...>::type; + + // Runs a vectorized function given arguments tuple and three index sequences: + // - Index is the full set of 0 ... (N-1) argument indices; + // - VIndex is the subset of argument indices with vectorized parameters, letting us access + // vectorized arguments (anything not in this sequence is passed through) + // - BIndex is a incremental sequence (beginning at 0) of the same size as VIndex, so that + // we can store vectorized buffer_infos in an array (argument VIndex has its buffer at + // index BIndex in the array). + template object run( + typename vectorize_arg::type &...args, + index_sequence i_seq, index_sequence vi_seq, index_sequence bi_seq) { + + // Pointers to values the function was called with; the vectorized ones set here will start + // out as array_t pointers, but they will be changed them to T pointers before we make + // call the wrapped function. Non-vectorized pointers are left as-is. + std::array params{{ &args... }}; + + // The array of `buffer_info`s of vectorized arguments: + std::array buffers{{ reinterpret_cast(params[VIndex])->request()... }}; /* Determine dimensions parameters of output array */ ssize_t nd = 0; @@ -1433,61 +1475,79 @@ struct vectorize_helper { auto trivial = broadcast(buffers, nd, shape); size_t ndim = (size_t) nd; - ssize_t size = 1; - std::vector strides(ndim); - if (ndim > 0) { - if (trivial == broadcast_trivial::f_trivial) { - strides[0] = sizeof(Return); - for (size_t i = 1; i < ndim; ++i) { - strides[i] = strides[i - 1] * shape[i - 1]; - size *= shape[i - 1]; - } - size *= shape[ndim - 1]; - } - else { - strides[ndim-1] = sizeof(Return); - for (size_t i = ndim - 1; i > 0; --i) { - strides[i - 1] = strides[i] * shape[i]; - size *= shape[i]; - } - size *= shape[0]; - } + size_t size = std::accumulate(shape.begin(), shape.end(), (size_t) 1, std::multiplies()); + + // If all arguments are 0-dimension arrays (i.e. single values) return a plain value (i.e. + // not wrapped in an array). + if (size == 1 && ndim == 0) { + PYBIND11_EXPAND_SIDE_EFFECTS(params[VIndex] = buffers[BIndex].ptr); + return cast(f(*reinterpret_cast *>(params[Index])...)); } - if (size == 1) - return cast(f(*reinterpret_cast(buffers[Index].ptr)...)); + array_t result; + if (trivial == broadcast_trivial::f_trivial) result = array_t(shape); + else result = array_t(shape); - array_t result(shape, strides); - auto buf = result.request(); - auto output = (Return *) buf.ptr; + if (size == 0) return result; /* Call the function */ - if (trivial == broadcast_trivial::non_trivial) { - apply_broadcast(buffers, buf, index); - } else { - for (ssize_t i = 0; i < size; ++i) - output[i] = f((reinterpret_cast(buffers[Index].ptr)[buffers[Index].size == 1 ? 0 : i])...); - } + if (trivial == broadcast_trivial::non_trivial) + apply_broadcast(buffers, params, result, i_seq, vi_seq, bi_seq); + else + apply_trivial(buffers, params, result.mutable_data(), size, i_seq, vi_seq, bi_seq); return result; } - template - void apply_broadcast(const std::array &buffers, - buffer_info &output, index_sequence) { - using input_iterator = multi_array_iterator; - using output_iterator = array_iterator; + template + void apply_trivial(std::array &buffers, + std::array ¶ms, + Return *out, + size_t size, + index_sequence, index_sequence, index_sequence) { + + // Initialize an array of mutable byte references and sizes with references set to the + // appropriate pointer in `params`; as we iterate, we'll increment each pointer by its size + // (except for singletons, which get an increment of 0). + std::array, NVectorized> vecparams{{ + std::pair( + reinterpret_cast(params[VIndex] = buffers[BIndex].ptr), + buffers[BIndex].size == 1 ? 0 : sizeof(param_n_t) + )... + }}; + + for (size_t i = 0; i < size; ++i) { + out[i] = f(*reinterpret_cast *>(params[Index])...); + for (auto &x : vecparams) x.first += x.second; + } + } + + template + void apply_broadcast(std::array &buffers, + std::array ¶ms, + array_t &output_array, + index_sequence, index_sequence, index_sequence) { - input_iterator input_iter(buffers, output.shape); - output_iterator output_end = array_end(output); + buffer_info output = output_array.request(); + multi_array_iterator input_iter(buffers, output.shape); - for (output_iterator iter = array_begin(output); - iter != output_end; ++iter, ++input_iter) { - *iter = f((input_iter.template data())...); + for (array_iterator iter = array_begin(output), end = array_end(output); + iter != end; + ++iter, ++input_iter) { + PYBIND11_EXPAND_SIDE_EFFECTS(( + params[VIndex] = input_iter.template data() + )); + *iter = f(*reinterpret_cast *>(std::get(params))...); } } }; +template +vectorize_helper +vectorize_extractor(const Func &f, Return (*) (Args ...)) { + return detail::vectorize_helper(f); +} + template struct handle_type_name> { static PYBIND11_DESCR name() { return _("numpy.ndarray[") + npy_format_descriptor::name() + _("]"); @@ -1496,22 +1556,32 @@ template struct handle_type_name> { NAMESPACE_END(detail) -template -detail::vectorize_helper -vectorize(const Func &f, Return (*) (Args ...)) { - return detail::vectorize_helper(f); -} - +// Vanilla pointer vectorizer: template -detail::vectorize_helper +detail::vectorize_helper vectorize(Return (*f) (Args ...)) { - return vectorize(f, f); + return detail::vectorize_helper(f); } -template ::type::operator())>::type> +// lambda vectorizer: +template ::operator())>::type> auto vectorize(Func &&f) -> decltype( - vectorize(std::forward(f), (FuncType *) nullptr)) { - return vectorize(std::forward(f), (FuncType *) nullptr); + detail::vectorize_extractor(std::forward(f), (FuncType *) nullptr)) { + return detail::vectorize_extractor(std::forward(f), (FuncType *) nullptr); +} + +// Vectorize a class method (non-const): +template ())), Return, Class *, Args...>> +Helper vectorize(Return (Class::*f)(Args...)) { + return Helper(std::mem_fn(f)); +} + +// Vectorize a class method (non-const): +template ())), Return, const Class *, Args...>> +Helper vectorize(Return (Class::*f)(Args...) const) { + return Helper(std::mem_fn(f)); } NAMESPACE_END(pybind11) -- cgit v1.2.3 From 24dec80b44b5980f348e99b410893fb139836965 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Wed, 31 May 2017 16:44:41 +0200 Subject: Help CLion IDE evaluate PYBIND11_NUMPY_DTYPE CLion slows to a crawl when evaluating the intricate `PYBIND11_NUMPY_DTYPE` macro. This commit replaces the macro cascade with a simple `(void)0` to ease IDE evaluation. --- include/pybind11/numpy.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 1e12ede..388e212 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1147,6 +1147,11 @@ private: } }; +#ifdef __CLION_IDE__ // replace heavy macro with dummy code for the IDE (doesn't affect code) +# define PYBIND11_NUMPY_DTYPE(Type, ...) ((void)0) +# define PYBIND11_NUMPY_DTYPE_EX(Type, ...) ((void)0) +#else + #define PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, Name) \ ::pybind11::detail::field_descriptor { \ Name, offsetof(T, Field), sizeof(decltype(std::declval().Field)), \ @@ -1214,6 +1219,8 @@ private: ::pybind11::detail::npy_format_descriptor::register_dtype \ ({PYBIND11_MAP2_LIST (PYBIND11_FIELD_DESCRIPTOR_EX, Type, __VA_ARGS__)}) +#endif // __CLION_IDE__ + template using array_iterator = typename std::add_pointer::type; -- cgit v1.2.3 From eb0f1cc7bfdb957d9e60f93ea91af9d56e3351a7 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 25 Jul 2017 16:22:45 -0400 Subject: Only allow unchecked()/mutable_unchecked() on an lvalue This should mitigate accidental invocation on a temporary array. Fixes #961. --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 388e212..57d4421 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -689,7 +689,7 @@ public: * care: the array must not be destroyed or reshaped for the duration of the returned object, * and the caller must take care not to access invalid dimensions or dimension indices. */ - template detail::unchecked_mutable_reference mutable_unchecked() { + template detail::unchecked_mutable_reference mutable_unchecked() & { if (Dims >= 0 && ndim() != Dims) throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -703,7 +703,7 @@ public: * reshaped for the duration of the returned object, and the caller must take care not to access * invalid dimensions or dimension indices. */ - template detail::unchecked_reference unchecked() const { + template detail::unchecked_reference unchecked() const & { if (Dims >= 0 && ndim() != Dims) throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -876,7 +876,7 @@ public: * care: the array must not be destroyed or reshaped for the duration of the returned object, * and the caller must take care not to access invalid dimensions or dimension indices. */ - template detail::unchecked_mutable_reference mutable_unchecked() { + template detail::unchecked_mutable_reference mutable_unchecked() & { return array::mutable_unchecked(); } @@ -887,7 +887,7 @@ public: * for the duration of the returned object, and the caller must take care not to access invalid * dimensions or dimension indices. */ - template detail::unchecked_reference unchecked() const { + template detail::unchecked_reference unchecked() const & { return array::unchecked(); } -- cgit v1.2.3 From a859dd67a29372e478ad862d8d2979930471532e Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 10 Aug 2017 12:03:29 -0400 Subject: Force hidden visibility on pybind code This adds a PYBIND11_NAMESPACE macro that expands to the `pybind11` namespace with hidden visibility under gcc-type compilers, and otherwise to the plain `pybind11`. This then forces hidden visibility on everything in pybind, solving the visibility issues discussed at end end of #949. --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 57d4421..4ee2561 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -34,7 +34,7 @@ upon the library user. */ static_assert(sizeof(ssize_t) == sizeof(Py_intptr_t), "ssize_t != Py_intptr_t"); -NAMESPACE_BEGIN(pybind11) +NAMESPACE_BEGIN(PYBIND11_NAMESPACE) class array; // Forward declaration @@ -1591,7 +1591,7 @@ Helper vectorize(Return (Class::*f)(Args...) const) { return Helper(std::mem_fn(f)); } -NAMESPACE_END(pybind11) +NAMESPACE_END(PYBIND11_NAMESPACE) #if defined(_MSC_VER) #pragma warning(pop) -- cgit v1.2.3 From b4bf5ed575da01e0846bc1f465443b2335219b53 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Wed, 10 May 2017 01:51:08 -0400 Subject: Added metatypes for dealing with functions/lambdas `function_signature_t` extracts the function type from a function, function pointer, or lambda. `is_lambda` (which is really `is_not_a_function_or_pointer_or_member_pointer`, but that name is a bit too long) checks whether the type is (in the approprate context) a lambda. `is_function_pointer` checks whether the type is a pointer to a function. --- include/pybind11/numpy.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 4ee2561..4bea460 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1571,10 +1571,10 @@ vectorize(Return (*f) (Args ...)) { } // lambda vectorizer: -template ::operator())>::type> +template ::value, int> = 0> auto vectorize(Func &&f) -> decltype( - detail::vectorize_extractor(std::forward(f), (FuncType *) nullptr)) { - return detail::vectorize_extractor(std::forward(f), (FuncType *) nullptr); + detail::vectorize_extractor(std::forward(f), (detail::function_signature_t *) nullptr)) { + return detail::vectorize_extractor(std::forward(f), (detail::function_signature_t *) nullptr); } // Vectorize a class method (non-const): -- cgit v1.2.3 From e9bb843edca5aed30eae341a8903998782c42679 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 22 Aug 2017 10:37:51 -0400 Subject: Fix clang5 warnings --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 4bea460..ce8c329 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -342,7 +342,7 @@ public: * dimensionality, this is not checked (and so is up to the caller to use safely). */ template const T &operator()(Ix... index) const { - static_assert(sizeof...(Ix) == Dims || Dynamic, + static_assert(ssize_t{sizeof...(Ix)} == Dims || Dynamic, "Invalid number of indices for unchecked array reference"); return *reinterpret_cast(data_ + byte_offset_unsafe(strides_, ssize_t(index)...)); } @@ -391,7 +391,7 @@ class unchecked_mutable_reference : public unchecked_reference { public: /// Mutable, unchecked access to data at the given indices. template T& operator()(Ix... index) { - static_assert(sizeof...(Ix) == Dims || Dynamic, + static_assert(ssize_t{sizeof...(Ix)} == Dims || Dynamic, "Invalid number of indices for unchecked array reference"); return const_cast(ConstBase::operator()(index...)); } -- cgit v1.2.3 From b0a0e4a23cfb82c92403012e4dc446928bb4347a Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Mon, 4 Sep 2017 21:16:09 +0200 Subject: Fix compilation with Clang on host GCC < 5 (old libstdc++) --- include/pybind11/numpy.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index ce8c329..55bb816 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -288,7 +288,9 @@ template using remove_all_extents_t = typename array_info::type; template using is_pod_struct = all_of< std::is_standard_layout, // since we're accessing directly in memory we need a standard layout type -#if !defined(__GNUG__) || defined(__clang__) || __GNUC__ >= 5 +#if !defined(__GNUG__) || defined(_LIBCPP_VERSION) || defined(_GLIBCXX_USE_CXX11_ABI) + // _GLIBCXX_USE_CXX11_ABI indicates that we're using libstdc++ from GCC 5 or newer, independent + // of the actual compiler (Clang can also use libstdc++, but it always defines __GNUC__ == 4). std::is_trivially_copyable, #else // GCC 4 doesn't implement is_trivially_copyable, so approximate it -- cgit v1.2.3 From c10ac6cf1f69ebe2b7dd26957325ea3b161cb7ff Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Sun, 2 Jul 2017 11:48:56 +0200 Subject: Make it possible to generate constexpr signatures in C++11 mode The current C++14 constexpr signatures don't require relaxed constexpr, but only `auto` return type deduction. To get around this in C++11, the type caster's `name()` static member functions are turned into `static constexpr auto` variables. --- include/pybind11/numpy.h | 74 +++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 35 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 55bb816..593a796 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -250,7 +250,7 @@ template struct array_info_scalar { typedef T type; static constexpr bool is_array = false; static constexpr bool is_empty = false; - static PYBIND11_DESCR extents() { return _(""); } + static constexpr auto extents = _(""); static void append_extents(list& /* shape */) { } }; // Computes underlying type and a comma-separated list of extents for array @@ -269,15 +269,9 @@ template struct array_info> { array_info::append_extents(shape); } - template::is_array, int> = 0> - static PYBIND11_DESCR extents() { - return _(); - } - - template::is_array, int> = 0> - static PYBIND11_DESCR extents() { - return concat(_(), array_info::extents()); - } + static constexpr auto extents = _::is_array>( + concat(_(), array_info::extents), _() + ); }; // For numpy we have special handling for arrays of characters, so we don't include // the size in the array extents. @@ -947,7 +941,7 @@ template struct format_descriptor::is_array>> { static std::string format() { using detail::_; - PYBIND11_DESCR extents = _("(") + detail::array_info::extents() + _(")"); + constexpr auto extents = _("(") + detail::array_info::extents + _(")"); return extents.text() + format_descriptor>::format(); } }; @@ -967,7 +961,7 @@ struct pyobject_caster> { static handle cast(const handle &src, return_value_policy /* policy */, handle /* parent */) { return src.inc_ref(); } - PYBIND11_TYPE_CASTER(type, handle_type_name::name()); + PYBIND11_TYPE_CASTER(type, handle_type_name::name); }; template @@ -977,7 +971,34 @@ struct compare_buffer_info::valu } }; -template struct npy_format_descriptor::value>> { +template +struct npy_format_descriptor_name; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = _::value>( + _("bool"), _::value>("int", "uint") + _() + ); +}; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = _::value || std::is_same::value>( + _("float") + _(), _("longdouble") + ); +}; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = _::value + || std::is_same::value>( + _("complex") + _(), _("longcomplex") + ); +}; + +template +struct npy_format_descriptor::value>> + : npy_format_descriptor_name { private: // NB: the order here must match the one in common.h constexpr static const int values[15] = { @@ -996,25 +1017,10 @@ public: return reinterpret_borrow(ptr); pybind11_fail("Unsupported buffer format!"); } - template ::value, int> = 0> - static PYBIND11_DESCR name() { - return _::value>(_("bool"), - _::value>("int", "uint") + _()); - } - template ::value, int> = 0> - static PYBIND11_DESCR name() { - return _::value || std::is_same::value>( - _("float") + _(), _("longdouble")); - } - template ::value, int> = 0> - static PYBIND11_DESCR name() { - return _::value || std::is_same::value>( - _("complex") + _(), _("longcomplex")); - } }; #define PYBIND11_DECL_CHAR_FMT \ - static PYBIND11_DESCR name() { return _("S") + _(); } \ + static constexpr auto name = _("S") + _(); \ static pybind11::dtype dtype() { return pybind11::dtype(std::string("S") + std::to_string(N)); } template struct npy_format_descriptor { PYBIND11_DECL_CHAR_FMT }; template struct npy_format_descriptor> { PYBIND11_DECL_CHAR_FMT }; @@ -1026,7 +1032,7 @@ private: public: static_assert(!array_info::is_empty, "Zero-sized arrays are not supported"); - static PYBIND11_DESCR name() { return _("(") + array_info::extents() + _(")") + base_descr::name(); } + static constexpr auto name = _("(") + array_info::extents + _(")") + base_descr::name; static pybind11::dtype dtype() { list shape; array_info::append_extents(shape); @@ -1038,7 +1044,7 @@ template struct npy_format_descriptor private: using base_descr = npy_format_descriptor::type>; public: - static PYBIND11_DESCR name() { return base_descr::name(); } + static constexpr auto name = base_descr::name; static pybind11::dtype dtype() { return base_descr::dtype(); } }; @@ -1113,7 +1119,7 @@ inline PYBIND11_NOINLINE void register_structured_dtype( template struct npy_format_descriptor { static_assert(is_pod_struct::value, "Attempt to use a non-POD or unimplemented POD type as a numpy dtype"); - static PYBIND11_DESCR name() { return make_caster::name(); } + static constexpr auto name = make_caster::name; static pybind11::dtype dtype() { return reinterpret_borrow(dtype_ptr()); @@ -1558,9 +1564,7 @@ vectorize_extractor(const Func &f, Return (*) (Args ...)) { } template struct handle_type_name> { - static PYBIND11_DESCR name() { - return _("numpy.ndarray[") + npy_format_descriptor::name() + _("]"); - } + static constexpr auto name = _("numpy.ndarray[") + npy_format_descriptor::name + _("]"); }; NAMESPACE_END(detail) -- cgit v1.2.3 From 56613945aea9f2786ddf004cf17a770dc5270c16 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Sun, 2 Jul 2017 12:52:00 +0200 Subject: Use semi-constexpr signatures on MSVC MSCV does not allow `&typeid(T)` in constexpr contexts, but the string part of the type signature can still be constexpr. In order to avoid `typeid` as long as possible, `descr` is modified to collect type information as template parameters instead of constexpr `typeid`. The actual `std::type_info` pointers are only collected in the end, as a `constexpr` (gcc/clang) or regular (MSVC) function call. Not only does it significantly reduce binary size on MSVC, gcc/clang benefit a little bit as well, since they can skip some intermediate `std::type_info*` arrays. --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 593a796..3755e89 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -941,8 +941,8 @@ template struct format_descriptor::is_array>> { static std::string format() { using detail::_; - constexpr auto extents = _("(") + detail::array_info::extents + _(")"); - return extents.text() + format_descriptor>::format(); + static constexpr auto extents = _("(") + detail::array_info::extents + _(")"); + return extents.text + format_descriptor>::format(); } }; -- cgit v1.2.3 From c6a57c10d10a458db1dfd8430d817017629f8b0a Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 19 Sep 2017 22:12:46 -0300 Subject: Fix dtype string leak `PyArray_DescrConverter_` doesn't steal a reference to the argument, and so the passed arguments shouldn't be `.release()`d. --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 3755e89..6fd8fdf 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -440,7 +440,7 @@ public: /// This is essentially the same as calling numpy.dtype(args) in Python. static dtype from_args(object args) { PyObject *ptr = nullptr; - if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr) + if (!detail::npy_api::get().PyArray_DescrConverter_(args.ptr(), &ptr) || !ptr) throw error_already_set(); return reinterpret_steal(ptr); } -- cgit v1.2.3 From d1db2ccfdf7e9c365fc34177f79ad742cf8e0d7c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Wed, 27 Dec 2017 15:00:27 +0000 Subject: Make register_dtype() accept any field containers (#1225) * Make register_dtype() accept any field containers * Add a test for programmatic dtype registration --- include/pybind11/numpy.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6fd8fdf..f64084e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -18,9 +18,9 @@ #include #include #include -#include #include #include +#include #include #if defined(_MSC_VER) @@ -1057,7 +1057,7 @@ struct field_descriptor { }; inline PYBIND11_NOINLINE void register_structured_dtype( - const std::initializer_list& fields, + any_container fields, const std::type_info& tinfo, ssize_t itemsize, bool (*direct_converter)(PyObject *, void *&)) { @@ -1066,7 +1066,7 @@ inline PYBIND11_NOINLINE void register_structured_dtype( pybind11_fail("NumPy: dtype is already registered"); list names, formats, offsets; - for (auto field : fields) { + for (auto field : *fields) { if (!field.descr) pybind11_fail(std::string("NumPy: unsupported field dtype: `") + field.name + "` @ " + tinfo.name()); @@ -1083,7 +1083,7 @@ inline PYBIND11_NOINLINE void register_structured_dtype( // - https://github.com/numpy/numpy/pull/7798 // Because of this, we won't use numpy's logic to generate buffer format // strings and will just do it ourselves. - std::vector ordered_fields(fields); + std::vector ordered_fields(std::move(fields)); std::sort(ordered_fields.begin(), ordered_fields.end(), [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); ssize_t offset = 0; @@ -1130,8 +1130,8 @@ template struct npy_format_descriptor { return format_str; } - static void register_dtype(const std::initializer_list& fields) { - register_structured_dtype(fields, typeid(typename std::remove_cv::type), + static void register_dtype(any_container fields) { + register_structured_dtype(std::move(fields), typeid(typename std::remove_cv::type), sizeof(T), &direct_converter); } @@ -1204,7 +1204,8 @@ private: #define PYBIND11_NUMPY_DTYPE(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ - ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + (::std::vector<::pybind11::detail::field_descriptor> \ + {PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) #ifdef _MSC_VER #define PYBIND11_MAP2_LIST_NEXT1(test, next) \ @@ -1225,7 +1226,8 @@ private: #define PYBIND11_NUMPY_DTYPE_EX(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ - ({PYBIND11_MAP2_LIST (PYBIND11_FIELD_DESCRIPTOR_EX, Type, __VA_ARGS__)}) + (::std::vector<::pybind11::detail::field_descriptor> \ + {PYBIND11_MAP2_LIST (PYBIND11_FIELD_DESCRIPTOR_EX, Type, __VA_ARGS__)}) #endif // __CLION_IDE__ -- cgit v1.2.3 From add56ccdcac23a6c522a2c1174a866e293c61dab Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 12 Jan 2018 12:37:54 -0400 Subject: MSVC workaround for broken `using detail::_` warning --- include/pybind11/numpy.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index f64084e..c7cbb9f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -940,9 +940,9 @@ struct format_descriptor::value>> { template struct format_descriptor::is_array>> { static std::string format() { - using detail::_; - static constexpr auto extents = _("(") + detail::array_info::extents + _(")"); - return extents.text + format_descriptor>::format(); + using namespace detail; + static constexpr auto extents = _("(") + array_info::extents + _(")"); + return extents.text + format_descriptor>::format(); } }; -- cgit v1.2.3 From 5ef1af138dc3ef94c05274ae554e70d64cd589bf Mon Sep 17 00:00:00 2001 From: Naotoshi Seo Date: Sun, 6 May 2018 13:59:25 +0000 Subject: Fix SEGV to create empty shaped numpy array (#1371) Fix a segfault when creating a 0-dimension, c-strides array. --- include/pybind11/numpy.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index c7cbb9f..0d92af2 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -758,8 +758,9 @@ protected: static std::vector c_strides(const std::vector &shape, ssize_t itemsize) { auto ndim = shape.size(); std::vector strides(ndim, itemsize); - for (size_t i = ndim - 1; i > 0; --i) - strides[i - 1] = strides[i] * shape[i]; + if (ndim > 0) + for (size_t i = ndim - 1; i > 0; --i) + strides[i - 1] = strides[i] * shape[i]; return strides; } -- cgit v1.2.3 From 7bb1da969a2747c24e6bd29dad6d44c4c5796141 Mon Sep 17 00:00:00 2001 From: Matthias Geier Date: Wed, 15 Aug 2018 17:13:36 +0200 Subject: fix copy-paste error: non-const -> const --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0d92af2..bdc3a5d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1593,7 +1593,7 @@ Helper vectorize(Return (Class::*f)(Args...)) { return Helper(std::mem_fn(f)); } -// Vectorize a class method (non-const): +// Vectorize a class method (const): template ())), Return, const Class *, Args...>> Helper vectorize(Return (Class::*f)(Args...) const) { -- cgit v1.2.3 From 0ca6867e8e233b64de24d5a01ab854702a1dbf66 Mon Sep 17 00:00:00 2001 From: Boris Staletic Date: Thu, 3 Jan 2019 12:02:39 +0100 Subject: Avoid Visual Studio 2017 15.9.4 ICE --- include/pybind11/numpy.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index bdc3a5d..37471d8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1466,7 +1466,10 @@ public: private: remove_reference_t f; - template using param_n_t = typename pack_element::call_type...>::type; + // Internal compiler error in MSVC 19.16.27025.1 (Visual Studio 2017 15.9.4), when compiling with "/permissive-" flag + // when arg_call_types is manually inlined. + using arg_call_types = std::tuple::call_type...>; + template using param_n_t = typename std::tuple_element::type; // Runs a vectorized function given arguments tuple and three index sequences: // - Index is the full set of 0 ... (N-1) argument indices; -- cgit v1.2.3 From ae951ca085f6a3a12958467b60ad0162eec7f72f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 6 Apr 2019 19:09:39 +0200 Subject: CI fixes (#1744) * Fix warning that not including a cmake source or build dir will be a fatal error (it is now on newest CMakes) * Fixes appveyor * Travis uses CMake 3.9 for more than a year now * Travis dropped sudo: false in December * Dropping Sphinx 2 - clang7: Suppress self-assign warnings; fix missing virtual dtors - pypy: - Keep old version (newer stuff breaks) - Pin packages to extra index for speed - travis: - Make docker explicit; remove docker if not needed - Make commands more verbose (for debugging / repro) - Make Ubuntu dist explicit per job - Fix Windows - Add names to travis --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 37471d8..ca954f5 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1509,7 +1509,7 @@ private: if (trivial == broadcast_trivial::f_trivial) result = array_t(shape); else result = array_t(shape); - if (size == 0) return result; + if (size == 0) return std::move(result); /* Call the function */ if (trivial == broadcast_trivial::non_trivial) @@ -1517,7 +1517,7 @@ private: else apply_trivial(buffers, params, result.mutable_data(), size, i_seq, vi_seq, bi_seq); - return result; + return std::move(result); } template -- cgit v1.2.3 From 9bb3313162c0b856125e481ceece9d8faa567716 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 7 Apr 2019 10:38:10 +0200 Subject: Fixing warnings about conversions in GCC 7+ (#1753) --- include/pybind11/numpy.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index ca954f5..b2a02e0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -855,14 +855,14 @@ public: // Reference to element at a given index template const T& at(Ix... index) const { - if (sizeof...(index) != ndim()) + if ((ssize_t) sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); return *(static_cast(array::data()) + byte_offset(ssize_t(index)...) / itemsize()); } // Mutable reference to element at a given index template T& mutable_at(Ix... index) { - if (sizeof...(index) != ndim()) + if ((ssize_t) sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); return *(static_cast(array::mutable_data()) + byte_offset(ssize_t(index)...) / itemsize()); } -- cgit v1.2.3 From a301c5add866fe91bf3b91df3d8034c124858a5e Mon Sep 17 00:00:00 2001 From: Igor Socec Date: Mon, 15 Jul 2019 12:31:03 +0100 Subject: Dtype field ordering for NumPy 1.14 (#1837) * Test dtype field order in numpy dtype tests When running tests with NumPy 1.14 or later this test exposes the "invalid buffer descriptor" error reported in #1274. * Create dtype_ptr with ordered fields --- include/pybind11/numpy.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index b2a02e0..a0441ef 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1066,8 +1066,14 @@ inline PYBIND11_NOINLINE void register_structured_dtype( if (numpy_internals.get_type_info(tinfo, false)) pybind11_fail("NumPy: dtype is already registered"); + // Use ordered fields because order matters as of NumPy 1.14: + // https://docs.scipy.org/doc/numpy/release.html#multiple-field-indexing-assignment-of-structured-arrays + std::vector ordered_fields(std::move(fields)); + std::sort(ordered_fields.begin(), ordered_fields.end(), + [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); + list names, formats, offsets; - for (auto field : *fields) { + for (auto& field : ordered_fields) { if (!field.descr) pybind11_fail(std::string("NumPy: unsupported field dtype: `") + field.name + "` @ " + tinfo.name()); @@ -1084,9 +1090,6 @@ inline PYBIND11_NOINLINE void register_structured_dtype( // - https://github.com/numpy/numpy/pull/7798 // Because of this, we won't use numpy's logic to generate buffer format // strings and will just do it ourselves. - std::vector ordered_fields(std::move(fields)); - std::sort(ordered_fields.begin(), ordered_fields.end(), - [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); ssize_t offset = 0; std::ostringstream oss; // mark the structure as unaligned with '^', because numpy and C++ don't -- cgit v1.2.3 From 4a3464fd88ae8666a670a17b99035beea6b464a8 Mon Sep 17 00:00:00 2001 From: Eric Cousineau Date: Sat, 1 Dec 2018 07:31:44 -0500 Subject: numpy: Provide concrete size aliases Test for dtype checks now succeed without warnings --- include/pybind11/numpy.h | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a0441ef..e67d371 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -108,6 +109,18 @@ inline numpy_internals& get_numpy_internals() { return *ptr; } +template struct same_size { + template using as = bool_constant; +}; + +// Lookup a type according to its size, and return a value corresponding to the NumPy typenum. +template +constexpr int platform_lookup(Int... codes) { + using code_index = std::integral_constant::template as, Check...>()>; + static_assert(code_index::value != sizeof...(Check), "Unable to match type on this platform"); + return std::get(std::make_tuple(codes...)); +} + struct npy_api { enum constants { NPY_ARRAY_C_CONTIGUOUS_ = 0x0001, @@ -126,7 +139,23 @@ struct npy_api { NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, NPY_OBJECT_ = 17, - NPY_STRING_, NPY_UNICODE_, NPY_VOID_ + NPY_STRING_, NPY_UNICODE_, NPY_VOID_, + // Platform-dependent normalization + NPY_INT8_ = NPY_BYTE_, + NPY_UINT8_ = NPY_UBYTE_, + NPY_INT16_ = NPY_SHORT_, + NPY_UINT16_ = NPY_USHORT_, + // `npy_common.h` defines the integer aliases. In order, it checks: + // NPY_BITSOF_LONG, NPY_BITSOF_LONGLONG, NPY_BITSOF_INT, NPY_BITSOF_SHORT, NPY_BITSOF_CHAR + // and assigns the alias to the first matching size, so we should check in this order. + NPY_INT32_ = platform_lookup( + NPY_LONG_, NPY_INT_, NPY_SHORT_), + NPY_UINT32_ = platform_lookup( + NPY_ULONG_, NPY_UINT_, NPY_USHORT_), + NPY_INT64_ = platform_lookup( + NPY_LONG_, NPY_LONGLONG_, NPY_INT_), + NPY_UINT64_ = platform_lookup( + NPY_ULONG_, NPY_ULONGLONG_, NPY_UINT_), }; typedef struct { @@ -1004,8 +1033,8 @@ private: // NB: the order here must match the one in common.h constexpr static const int values[15] = { npy_api::NPY_BOOL_, - npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, - npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_, + npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_INT16_, npy_api::NPY_UINT16_, + npy_api::NPY_INT32_, npy_api::NPY_UINT32_, npy_api::NPY_INT64_, npy_api::NPY_UINT64_, npy_api::NPY_FLOAT_, npy_api::NPY_DOUBLE_, npy_api::NPY_LONGDOUBLE_, npy_api::NPY_CFLOAT_, npy_api::NPY_CDOUBLE_, npy_api::NPY_CLONGDOUBLE_ }; -- cgit v1.2.3 From c9d32a81f40ad540015814edf13b29980c63e39c Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jul 2019 09:35:32 +0000 Subject: numpy: fix refcount leak to dtype singleton (#1860) PyArray_DescrFromType returns a new reference, not borrowed one --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index e67d371..8b21d3d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1044,7 +1044,7 @@ public: static pybind11::dtype dtype() { if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) - return reinterpret_borrow(ptr); + return reinterpret_steal(ptr); pybind11_fail("Unsupported buffer format!"); } }; -- cgit v1.2.3 From 7f5dad7d5fba3fb6fccef966f7cde5ca24509473 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sat, 21 Sep 2019 18:54:10 +0200 Subject: Remove usage of C++14 constructs (fixes #1929) --- include/pybind11/numpy.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/pybind11/numpy.h') diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8b21d3d..ba41a22 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -113,12 +113,12 @@ template struct same_size { template using as = bool_constant; }; +template constexpr int platform_lookup() { return -1; } + // Lookup a type according to its size, and return a value corresponding to the NumPy typenum. -template -constexpr int platform_lookup(Int... codes) { - using code_index = std::integral_constant::template as, Check...>()>; - static_assert(code_index::value != sizeof...(Check), "Unable to match type on this platform"); - return std::get(std::make_tuple(codes...)); +template +constexpr int platform_lookup(int I, Ints... Is) { + return sizeof(Concrete) == sizeof(T) ? I : platform_lookup(Is...); } struct npy_api { -- cgit v1.2.3