diff -r 9b1daa80168d Doc/c-api/buffer.rst --- a/Doc/c-api/buffer.rst Sat Feb 12 01:03:31 2011 +0100 +++ b/Doc/c-api/buffer.rst Sun Feb 13 19:53:31 2011 +0100 @@ -78,6 +78,19 @@ around a buffer is needed, a :ref:`memoryview ` object can be created. +The exporter of the buffer is responsible for the management of any +resources needed for providing it, for example allocating and +freeing memory for the ``shape`` and ``strides`` fields. The exporter +of the buffer guarantees that the contents of :c:func:`Py_buffer` +structure (for example, shape and strides) do not change before +:c:func:`PyBuffer_Release` is called. The only exception is that the +content of the buffer pointed to by the :c:member:`buf` field may +change. + +The consumer must not modify the ``obj`` and ``internal`` fields of +the ``Py_buffer`` structure filled in by :c:func:`PyObject_GetBuffer`. +It must also make sure that these fields are intact when calling +:c:func:`PyBuffer_Release`. .. c:type:: Py_buffer @@ -85,6 +98,13 @@ A pointer to the start of the memory for the object. + .. c:member:: PyObject *obj + + A reference to the object exporting the buffer. + + The exporter should set this field to *self* and ``Py_INCREF`` it. + The consumer should not modify this field. + .. c:member:: Py_ssize_t len :noindex: @@ -98,8 +118,9 @@ :noindex: A *NULL* terminated string in :mod:`struct` module style syntax giving - the contents of the elements available through the buffer. If this is - *NULL*, ``"B"`` (unsigned bytes) is assumed. + the contents of the elements available through the buffer. + + If *format* is *NULL*, ``"B"`` (unsigned bytes) is assumed. .. c:member:: int ndim @@ -114,11 +135,17 @@ ``((*shape)[0] * ... * (*shape)[ndims-1])*itemsize`` should be equal to :c:data:`len`. + If *shape* is *NULL*, the buffer is 1-D and contains *len* bytes. + .. c:member:: Py_ssize_t *strides An array of :c:type:`Py_ssize_t`\s the length of :c:data:`ndim` giving the number of bytes to skip to get to a new element in each dimension. + If *strides* is *NULL*, the elements in the buffer are laid out + contiguously in memory, and for multidimensional arrays, in + C-order (row-major). + .. c:member:: Py_ssize_t *suboffsets An array of :c:type:`Py_ssize_t`\s the length of :c:data:`ndim`. If these @@ -128,6 +155,8 @@ suboffset value that it negative indicates that no de-referencing should occur (striding in a contiguous memory block). + If *suboffsets* is *NULL*, the buffer consists of a single segment. + Here is a function that returns a pointer to the element in an N-D array pointed to by an N-dimensional index when there are both non-NULL strides and suboffsets:: @@ -145,10 +174,9 @@ return (void*)pointer; } - .. c:member:: Py_ssize_t itemsize - This is a storage for the itemsize (in bytes) of each element of the + This is a storage for the item size (in bytes) of each element of the shared memory. It is technically un-necessary as it can be obtained using :c:func:`PyBuffer_SizeFromFormat`, however an exporter may know this information without parsing the format string and it is necessary @@ -157,11 +185,10 @@ .. c:member:: void *internal - This is for use internally by the exporting object. For example, this - might be re-cast as an integer by the exporter and used to store flags - about whether or not the shape, strides, and suboffsets arrays must be - freed when the buffer is released. The consumer should never alter this - value. + This is for internal use by the exporting object. + + The consumer should not modify this field. The exporter can use + it for any purpose. Buffer-related functions @@ -295,7 +322,7 @@ .. c:function:: void PyBuffer_Release(Py_buffer *view) Release the buffer *view*. This should be called when the buffer is no - longer being used as it may free memory from it. + longer being used, to free certain resources associated with it. .. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *) diff -r 9b1daa80168d Doc/c-api/typeobj.rst --- a/Doc/c-api/typeobj.rst Sat Feb 12 01:03:31 2011 +0100 +++ b/Doc/c-api/typeobj.rst Sun Feb 13 19:53:31 2011 +0100 @@ -1195,44 +1195,76 @@ .. sectionauthor:: Benjamin Peterson -The :ref:`buffer interface ` exports a model where an object can expose its internal -data. +The :ref:`buffer interface ` exports a model where an +object can expose its internal data. -If an object does not export the buffer interface, then its :attr:`tp_as_buffer` -member in the :c:type:`PyTypeObject` structure should be *NULL*. Otherwise, the -:attr:`tp_as_buffer` will point to a :c:type:`PyBufferProcs` structure. +If an object does not export the buffer interface, then its +:attr:`tp_as_buffer` member in the :c:type:`PyTypeObject` structure +should be *NULL*. Otherwise, the :attr:`tp_as_buffer` will point to a +:c:type:`PyBufferProcs` structure. .. c:type:: PyBufferProcs - Structure used to hold the function pointers which define an implementation of - the buffer protocol. + Structure used to hold the function pointers which define an + implementation of the buffer protocol. .. c:member:: getbufferproc bf_getbuffer - This should fill a :c:type:`Py_buffer` with the necessary data for - exporting the type. The signature of :data:`getbufferproc` is ``int - (PyObject *obj, Py_buffer *view, int flags)``. *obj* is the object to - export, *view* is the :c:type:`Py_buffer` struct to fill, and *flags* gives - the conditions the caller wants the memory under. (See - :c:func:`PyObject_GetBuffer` for all flags.) :c:member:`bf_getbuffer` is - responsible for filling *view* with the appropriate information. - (:c:func:`PyBuffer_FillView` can be used in simple cases.) See - :c:type:`Py_buffer`\s docs for what needs to be filled in. + This should fill a :c:type:`Py_buffer` with the necessary data + for exporting the type. The signature of :data:`getbufferproc` + is ``int (PyObject *obj, Py_buffer *view, int flags)``. *obj* + is the object to export, *view* is the :c:type:`Py_buffer` + struct to fill, and *flags* gives the conditions the caller + wants the memory under. (See :c:func:`PyObject_GetBuffer` for + all flags.) + :c:member:`bf_getbuffer` is responsible for filling *view* with + the appropriate information. See :c:type:`Py_buffer`\s docs for + a description of the different fields. Unused fields should be + set to NULL. :c:func:`PyBuffer_FillView` can be used in simple + cases. + + On success, the function should return 0. On error, it should + return -1 (in which case the :c:data:`view` object will never be + passed on to :c:data:`bf_releasbuffer`). + + The exporter of the buffer interface must make sure that any + memory pointed to in the :c:type:`Py_buffer` structure remains + valid until the buffer is released. Moreover, the exporter must + guarantee that the contents of the :c:type:`Py_buffer` structure + remain unchanged after returning from :c:member:`bf_getbuffer`. + Only changes in the contents of the memory buffer pointed to by + the ``buf`` field are allowed. (For example, the ``shape`` or + other properties of the buffer must not change once the + buffer is obtained.) + + The exporter is responsible for the management of any resources + needed for providing the buffer. For instance, it may be + necessary to allocate memory for the shape and stride arrays in + the :c:type:`Py_buffer` structure. To free such resources when + the buffer is released, the exporter can define the + :c:member:`bf_releasebuffer` method. .. c:member:: releasebufferproc bf_releasebuffer - This should release the resources of the buffer. The signature of - :c:data:`releasebufferproc` is ``void (PyObject *obj, Py_buffer *view)``. - If the :c:data:`bf_releasebuffer` function is not provided (i.e. it is - *NULL*), then it does not ever need to be called. + This should release the resources of the buffer. The signature + of :c:member:`releasebufferproc` is ``void (PyObject *obj, + Py_buffer *view)``. The :c:member:`bf_releasebuffer` can be + *NULL*, if no additional actions are necessary when the buffer + is released. - The exporter of the buffer interface must make sure that any memory - pointed to in the :c:type:`Py_buffer` structure remains valid until - releasebuffer is called. Exporters will need to define a - :c:data:`bf_releasebuffer` function if they can re-allocate their memory, - strides, shape, suboffsets, or format variables which they might share - through the struct bufferinfo. + Exporters can define a :c:member:`bf_releasebuffer` function if + they need to release some resources dynamically allocated for + each buffer (for instance, memory containing the stride or shape + arrays). + + The :c:member:`bf_releasebuffer` is called exactly once per + buffer. The exporter can assume that the :c:member:`internal` + field of the :c:type:`Py_buffer` contains data filled by a + corresponding previous call to :c:member:`bf_getbuffer`. The + :c:member:`bf_releasebuffer` routine may not make any other + assumptions about the contents of the structure, as it may be + modified and copied by the consumer. See :c:func:`PyBuffer_Release`. diff -r 9b1daa80168d Objects/memoryobject.c --- a/Objects/memoryobject.c Sat Feb 12 01:03:31 2011 +0100 +++ b/Objects/memoryobject.c Sun Feb 13 19:53:31 2011 +0100 @@ -33,30 +33,70 @@ return -1; } +/* Make a shallow copy of the Py_buffer structure */ static void dup_buffer(Py_buffer *dest, Py_buffer *src) { *dest = *src; - if (src->ndim == 1 && src->shape != NULL) { - dest->shape = &(dest->smalltable[0]); - dest->shape[0] = get_shape0(src); + + /* Relocate pointers in Py_buffer if they point inside the original + * structure (which can reside on the stack and go out of scope). + */ +#define RELOCATE_LOCAL_POINTER(field) \ + if ((char*)src->field >= (char*)src && \ + (char*)src->field < (char*)src + sizeof(Py_buffer)) { \ + dest->field = (Py_ssize_t*)((char*)dest \ + + ((char*)src->field - (char*)src)); \ } - if (src->ndim == 1 && src->strides != NULL) { - dest->strides = &(dest->smalltable[1]); - dest->strides[0] = src->strides[0]; - } + RELOCATE_LOCAL_POINTER(strides); + RELOCATE_LOCAL_POINTER(shape); + RELOCATE_LOCAL_POINTER(suboffsets); } static int memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) { - int res = 0; + void *orig_internal; CHECK_RELEASED_INT(self); - if (self->view.obj != NULL) + if (!view) { + PyErr_SetString(PyExc_BufferError, "NULL buffer info structure"); + return -1; + } + if (self->view.obj != NULL) { + int res; res = PyObject_GetBuffer(self->view.obj, view, flags); - if (view) - dup_buffer(view, &self->view); - return res; + if (res != 0) + return res; + } + + /* + Replace buffer info by whatever this memoryview has; however, don't + clobber the internal pointer. + */ + orig_internal = view->internal; + dup_buffer(view, &self->view); + view->internal = orig_internal; + + /* + TODO: this will cause crashes if the PyObject_GetBuffer above yields a + buffer different from the one when the memoryview was created. This is + not (and probably should not be) forbidden by the spec. + + Since this memoryview does not export the buffer we return here (view->obj + != self), `self->view` can be released before `view` is released. That + is, the buffer to which self->view.buf (and view->buf) points to can + become invalid before PyBuffer_Release is called on `view` => problems. + + The only reliable solution seems to have memory_getbuf to set view->obj = + self, and provide the buffer interface by itself. + */ + + /* + TODO: there is no guarantee that this information + matches what the caller specified in flags + */ + + return 0; } static void @@ -430,12 +470,12 @@ CHECK_RELEASED(mem); if (strcmp(view->format, "B") || view->itemsize != 1) { - PyErr_SetString(PyExc_NotImplementedError, + PyErr_SetString(PyExc_NotImplementedError, "tolist() only supports byte views"); return NULL; } if (view->ndim != 1) { - PyErr_SetString(PyExc_NotImplementedError, + PyErr_SetString(PyExc_NotImplementedError, "tolist() only supports one-dimensional objects"); return NULL; } @@ -522,6 +562,7 @@ memory_item(PyMemoryViewObject *self, Py_ssize_t result) { Py_buffer *view = &(self->view); + Py_ssize_t shape0; CHECK_RELEASED(self); if (view->ndim == 0) { @@ -533,10 +574,14 @@ /* Return a bytes object */ char *ptr; ptr = (char *)view->buf; + shape0 = get_shape0(view); + if (shape0 < 0) { + return NULL; + } if (result < 0) { - result += get_shape0(view); + result += shape0; } - if ((result < 0) || (result >= get_shape0(view))) { + if ((result < 0) || (result >= shape0)) { PyErr_SetString(PyExc_IndexError, "index out of bounds"); return NULL; @@ -572,7 +617,7 @@ { Py_buffer *view; view = &(self->view); - + CHECK_RELEASED(self); if (view->ndim == 0) { if (key == Py_Ellipsis || @@ -595,19 +640,25 @@ } else if (PySlice_Check(key)) { Py_ssize_t start, stop, step, slicelength; + Py_ssize_t shape0; - if (PySlice_GetIndicesEx(key, get_shape0(view), + shape0 = get_shape0(view); + if (shape0 < 0) { + return NULL; + } + + if (PySlice_GetIndicesEx(key, shape0, &start, &stop, &step, &slicelength) < 0) { return NULL; } - + if (step == 1 && view->ndim == 1) { Py_buffer newview; void *newbuf = (char *) view->buf + start * view->itemsize; int newflags = view->readonly ? PyBUF_CONTIG_RO : PyBUF_CONTIG; - + /* XXX There should be an API to create a subbuffer */ if (view->obj != NULL) { if (PyObject_GetBuffer(view->obj, &newview, newflags) == -1) @@ -628,7 +679,7 @@ return NULL; } PyErr_Format(PyExc_TypeError, - "cannot index memory using \"%.200s\"", + "cannot index memory using \"%.200s\"", key->ob_type->tp_name); return NULL; } @@ -638,7 +689,7 @@ static int memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) { - Py_ssize_t start, len, bytelen; + Py_ssize_t start, len, bytelen, shape0; Py_buffer srcview; Py_buffer *view = &(self->view); char *srcbuf, *destbuf; @@ -658,14 +709,18 @@ PyErr_SetNone(PyExc_NotImplementedError); return -1; } + shape0 = get_shape0(view); + if (shape0 < 0) { + return -1; + } if (PyIndex_Check(key)) { start = PyNumber_AsSsize_t(key, NULL); if (start == -1 && PyErr_Occurred()) return -1; if (start < 0) { - start += get_shape0(view); + start += shape0; } - if ((start < 0) || (start >= get_shape0(view))) { + if ((start < 0) || (start >= shape0)) { PyErr_SetString(PyExc_IndexError, "index out of bounds"); return -1; @@ -675,7 +730,7 @@ else if (PySlice_Check(key)) { Py_ssize_t stop, step; - if (PySlice_GetIndicesEx(key, get_shape0(view), + if (PySlice_GetIndicesEx(key, shape0, &start, &stop, &step, &len) < 0) { return -1; } @@ -686,7 +741,7 @@ } else { PyErr_Format(PyExc_TypeError, - "cannot index memory using \"%.200s\"", + "cannot index memory using \"%.200s\"", key->ob_type->tp_name); return -1; } @@ -698,7 +753,7 @@ (e.g. assign 2 shorts to a 4-byte slice) */ if (srcview.itemsize != view->itemsize) { PyErr_Format(PyExc_TypeError, - "mismatching item sizes for \"%.200s\" and \"%.200s\"", + "mismatching item sizes for \"%.200s\" and \"%.200s\"", view->obj->ob_type->tp_name, srcview.obj->ob_type->tp_name); goto _error; }