diff -r 38f75ebfe4cd Include/genobject.h --- a/Include/genobject.h Tue Jun 09 18:42:36 2015 +0300 +++ b/Include/genobject.h Tue Jun 09 12:35:35 2015 -0400 @@ -38,11 +38,10 @@ #define PyGen_Check(op) PyObject_TypeCheck(op, &PyGen_Type) #define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type) -#define PyGen_CheckCoroutineExact(op) (PyGen_CheckExact(op) && \ - (((PyCodeObject*) \ - ((PyGenObject*)op)->gi_code) \ - ->co_flags & (CO_ITERABLE_COROUTINE | \ - CO_COROUTINE))) +#define PyGen_IsCoroutine(op) (PyGen_CheckExact(op) && \ + (((PyCodeObject*) \ + ((PyGenObject*)op)->gi_code) \ + ->co_flags & CO_ITERABLE_COROUTINE)) PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *); PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(struct _frame *, @@ -54,6 +53,15 @@ PyObject *_PyGen_GetAwaitableIter(PyObject *o); +PyAPI_DATA(PyTypeObject) PyCoro_Type; + +#define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type) + +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) PyCoro_NewWithQualName(struct _frame *, + PyObject *name, PyObject *qualname); +#endif + #ifdef __cplusplus } #endif diff -r 38f75ebfe4cd Include/opcode.h --- a/Include/opcode.h Tue Jun 09 18:42:36 2015 +0300 +++ b/Include/opcode.h Tue Jun 09 12:35:35 2015 -0400 @@ -45,6 +45,7 @@ #define BINARY_OR 66 #define INPLACE_POWER 67 #define GET_ITER 68 +#define GET_YIELD_FROM_ITER 69 #define PRINT_EXPR 70 #define LOAD_BUILD_CLASS 71 #define YIELD_FROM 72 diff -r 38f75ebfe4cd Lib/_collections_abc.py --- a/Lib/_collections_abc.py Tue Jun 09 18:42:36 2015 +0300 +++ b/Lib/_collections_abc.py Tue Jun 09 12:35:35 2015 -0400 @@ -53,6 +53,11 @@ mappingproxy = type(type.__dict__) generator = type((lambda: (yield))()) +async def _coro(): pass +_coro = _coro() +coroutine = type(_coro) +_coro.close() # Prevent ResourceWarning + ### ONE-TRICK PONIES ### @@ -78,17 +83,12 @@ class _AwaitableMeta(ABCMeta): def __instancecheck__(cls, instance): - # 0x80 = CO_COROUTINE # 0x100 = CO_ITERABLE_COROUTINE # We don't want to import 'inspect' module, as # a dependency for 'collections.abc'. - CO_COROUTINES = 0x80 | 0x100 - if (isinstance(instance, generator) and - instance.gi_code.co_flags & CO_COROUTINES): - + instance.gi_code.co_flags & 0x100): return True - return super().__instancecheck__(instance) @@ -111,6 +111,9 @@ return NotImplemented +Awaitable.register(coroutine) + + class Coroutine(Awaitable): __slots__ = () @@ -159,6 +162,9 @@ return NotImplemented +Coroutine.register(coroutine) + + class AsyncIterable(metaclass=ABCMeta): __slots__ = () diff -r 38f75ebfe4cd Lib/opcode.py --- a/Lib/opcode.py Tue Jun 09 18:42:36 2015 +0300 +++ b/Lib/opcode.py Tue Jun 09 12:35:35 2015 -0400 @@ -103,6 +103,7 @@ def_op('BINARY_OR', 66) def_op('INPLACE_POWER', 67) def_op('GET_ITER', 68) +def_op('GET_YIELD_FROM_ITER', 69) def_op('PRINT_EXPR', 70) def_op('LOAD_BUILD_CLASS', 71) diff -r 38f75ebfe4cd Lib/test/test_coroutines.py --- a/Lib/test/test_coroutines.py Tue Jun 09 18:42:36 2015 +0300 +++ b/Lib/test/test_coroutines.py Tue Jun 09 12:35:35 2015 -0400 @@ -24,7 +24,7 @@ def run_async(coro): - assert coro.__class__ is types.GeneratorType + assert coro.__class__ in {types.GeneratorType, types.CoroutineType} buffer = [] result = None @@ -121,7 +121,7 @@ return 10 f = foo() - self.assertIsInstance(f, types.GeneratorType) + self.assertIsInstance(f, types.CoroutineType) self.assertTrue(bool(foo.__code__.co_flags & 0x80)) self.assertTrue(bool(foo.__code__.co_flags & 0x20)) self.assertTrue(bool(f.gi_code.co_flags & 0x80)) @@ -152,7 +152,7 @@ raise StopIteration check = lambda: self.assertRaisesRegex( - TypeError, "coroutine-objects do not support iteration") + TypeError, "'coroutine' object is not iterable") with check(): list(foo()) @@ -166,9 +166,6 @@ with check(): iter(foo()) - with check(): - next(foo()) - with silence_coro_gc(), check(): for i in foo(): pass @@ -185,7 +182,7 @@ await bar() check = lambda: self.assertRaisesRegex( - TypeError, "coroutine-objects do not support iteration") + TypeError, "'coroutine' object is not iterable") with check(): for el in foo(): pass diff -r 38f75ebfe4cd Lib/types.py --- a/Lib/types.py Tue Jun 09 18:42:36 2015 +0300 +++ b/Lib/types.py Tue Jun 09 12:35:35 2015 -0400 @@ -19,6 +19,11 @@ yield 1 GeneratorType = type(_g()) +async def _c(): pass +_c = _c() +CoroutineType = type(_c) +_c.close() # Prevent ResourceWarning + class _C: def _m(self): pass MethodType = type(_C()._m) @@ -40,7 +45,7 @@ GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) -del sys, _f, _g, _C, # Not for export +del sys, _f, _g, _C, _c, # Not for export # Provide a PEP 3115 compliant mechanism for class creation @@ -167,26 +172,30 @@ # We don't want to import 'dis' or 'inspect' just for # these constants. CO_GENERATOR = 0x20 + CO_COROUTINE = 0x80 CO_ITERABLE_COROUTINE = 0x100 if not callable(func): raise TypeError('types.coroutine() expects a callable') if (isinstance(func, FunctionType) and - isinstance(getattr(func, '__code__', None), CodeType) and - (func.__code__.co_flags & CO_GENERATOR)): + isinstance(getattr(func, '__code__', None), CodeType)): - # TODO: Implement this in C. - co = func.__code__ - func.__code__ = CodeType( - co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, - co.co_stacksize, - co.co_flags | CO_ITERABLE_COROUTINE, - co.co_code, - co.co_consts, co.co_names, co.co_varnames, co.co_filename, - co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars, - co.co_cellvars) - return func + if func.__code__.co_flags & CO_GENERATOR: + # TODO: Implement this in C. + co = func.__code__ + func.__code__ = CodeType( + co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, + co.co_stacksize, + co.co_flags | CO_ITERABLE_COROUTINE, + co.co_code, + co.co_consts, co.co_names, co.co_varnames, co.co_filename, + co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars, + co.co_cellvars) + return func + + if func.__code__.co_flags & CO_COROUTINE: + return func # The following code is primarily to support functions that # return generator-like objects (for instance generators @@ -218,6 +227,8 @@ @_functools.wraps(func) def wrapped(*args, **kwargs): coro = func(*args, **kwargs) + if coro.__class__ is CoroutineType: + return coro if coro.__class__ is GeneratorType: return GeneratorWrapper(coro) # slow checks diff -r 38f75ebfe4cd Objects/genobject.c --- a/Objects/genobject.c Tue Jun 09 18:42:36 2015 +0300 +++ b/Objects/genobject.c Tue Jun 09 12:35:35 2015 -0400 @@ -432,12 +432,6 @@ static PyObject * gen_iternext(PyGenObject *gen) { - if (((PyCodeObject*)gen->gi_code)->co_flags & CO_COROUTINE) { - PyErr_SetString(PyExc_TypeError, - "coroutine-objects do not support iteration"); - return NULL; - } - return gen_send_ex(gen, NULL, 0); } @@ -494,14 +488,8 @@ static PyObject * gen_repr(PyGenObject *gen) { - if (PyGen_CheckCoroutineExact(gen)) { - return PyUnicode_FromFormat("", - gen->gi_qualname, gen); - } - else { - return PyUnicode_FromFormat("", - gen->gi_qualname, gen); - } + return PyUnicode_FromFormat("", + gen->gi_qualname, gen); } static PyObject * @@ -540,12 +528,6 @@ static PyObject * gen_get_iter(PyGenObject *gen) { - if (((PyCodeObject*)gen->gi_code)->co_flags & CO_COROUTINE) { - PyErr_SetString(PyExc_TypeError, - "coroutine-objects do not support iteration"); - return NULL; - } - Py_INCREF(gen); return (PyObject *)gen; } @@ -645,10 +627,11 @@ _PyGen_Finalize, /* tp_finalize */ }; -PyObject * -PyGen_NewWithQualName(PyFrameObject *f, PyObject *name, PyObject *qualname) +static PyObject * +gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, + PyObject *name, PyObject *qualname) { - PyGenObject *gen = PyObject_GC_New(PyGenObject, &PyGen_Type); + PyGenObject *gen = PyObject_GC_New(PyGenObject, type); if (gen == NULL) { Py_DECREF(f); return NULL; @@ -674,9 +657,15 @@ } PyObject * +PyGen_NewWithQualName(PyFrameObject *f, PyObject *name, PyObject *qualname) +{ + return gen_new_with_qualname(&PyGen_Type, f, name, qualname); +} + +PyObject * PyGen_New(PyFrameObject *f) { - return PyGen_NewWithQualName(f, NULL, NULL); + return gen_new_with_qualname(&PyGen_Type, f, NULL, NULL); } int @@ -711,8 +700,8 @@ unaryfunc getter = NULL; PyTypeObject *ot; - if (PyGen_CheckCoroutineExact(o)) { - /* Fast path. It's a central function for 'await'. */ + if (PyCoro_CheckExact(o) || PyGen_IsCoroutine(o)) { + /* 'o' is a coroutine. */ Py_INCREF(o); return o; } @@ -724,22 +713,19 @@ if (getter != NULL) { PyObject *res = (*getter)(o); if (res != NULL) { - if (!PyIter_Check(res)) { + if (PyCoro_CheckExact(res) || PyGen_IsCoroutine(res)) { + /* __await__ must return an *iterator*, not + a coroutine or another awaitable (see PEP 492) */ + PyErr_SetString(PyExc_TypeError, + "__await__() returned a coroutine"); + Py_CLEAR(res); + } else if (!PyIter_Check(res)) { PyErr_Format(PyExc_TypeError, "__await__() returned non-iterator " "of type '%.100s'", Py_TYPE(res)->tp_name); Py_CLEAR(res); } - else { - if (PyGen_CheckCoroutineExact(res)) { - /* __await__ must return an *iterator*, not - a coroutine or another awaitable (see PEP 492) */ - PyErr_SetString(PyExc_TypeError, - "__await__() returned a coroutine"); - Py_CLEAR(res); - } - } } return res; } @@ -750,3 +736,94 @@ return NULL; } + +/* Coroutine Object */ + +static PyObject * +coro_repr(PyGenObject *gen) +{ + return PyUnicode_FromFormat("", + gen->gi_qualname, gen); +} + +static PyGetSetDef coro_getsetlist[] = { + {"__name__", (getter)gen_get_name, (setter)gen_set_name, + PyDoc_STR("name of the coroutine")}, + {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + PyDoc_STR("qualified name of the coroutine")}, + {NULL} /* Sentinel */ +}; + +static PyMemberDef coro_memberlist[] = { + {"gi_frame", T_OBJECT, offsetof(PyGenObject, gi_frame), READONLY}, + {"gi_running", T_BOOL, offsetof(PyGenObject, gi_running), READONLY}, + {"gi_code", T_OBJECT, offsetof(PyGenObject, gi_code), READONLY}, + {NULL} /* Sentinel */ +}; + +static PyMethodDef coro_methods[] = { + {"send",(PyCFunction)_PyGen_Send, METH_O, send_doc}, + {"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc}, + {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, + {NULL, NULL} /* Sentinel */ +}; + +PyTypeObject PyCoro_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "coroutine", /* tp_name */ + sizeof(PyGenObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)gen_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + (reprfunc)coro_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)gen_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(PyGenObject, gi_weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + coro_methods, /* tp_methods */ + coro_memberlist, /* tp_members */ + coro_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + _PyGen_Finalize, /* tp_finalize */ +}; + +PyObject * +PyCoro_NewWithQualName(PyFrameObject *f, PyObject *name, PyObject *qualname) +{ + return gen_new_with_qualname(&PyCoro_Type, f, name, qualname); +} diff -r 38f75ebfe4cd Python/ceval.c --- a/Python/ceval.c Tue Jun 09 18:42:36 2015 +0300 +++ b/Python/ceval.c Tue Jun 09 12:35:35 2015 -0400 @@ -2034,25 +2034,7 @@ PyObject *v = POP(); PyObject *reciever = TOP(); int err; - if (PyGen_CheckExact(reciever)) { - if ( - (((PyCodeObject*) \ - ((PyGenObject*)reciever)->gi_code)->co_flags & - CO_COROUTINE) - && !(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) - { - /* If we're yielding-from a coroutine object from a regular - generator object - raise an error. */ - - Py_CLEAR(v); - Py_CLEAR(reciever); - SET_TOP(NULL); - - PyErr_SetString(PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "from a generator"); - goto error; - } + if (PyGen_CheckExact(reciever) || PyCoro_CheckExact(reciever)) { retval = _PyGen_Send((PyGenObject *)reciever, v); } else { _Py_IDENTIFIER(send); @@ -2929,19 +2911,33 @@ TARGET(GET_ITER) { /* before: [obj]; after [getiter(obj)] */ PyObject *iterable = TOP(); + PyObject *iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + SET_TOP(iter); + if (iter == NULL) + goto error; + PREDICT(FOR_ITER); + DISPATCH(); + } + + TARGET(GET_YIELD_FROM_ITER) { + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable = TOP(); PyObject *iter; - /* If we have a generator object on top -- keep it there, - it's already an iterator. - - This is needed to allow use of 'async def' coroutines - in 'yield from' expression from generator-based coroutines - (decorated with types.coroutine()). - - 'yield from' is compiled to GET_ITER..YIELD_FROM combination, - but since coroutines raise TypeError in their 'tp_iter' we - need a way for them to "pass through" the GET_ITER. - */ - if (!PyGen_CheckExact(iterable)) { + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + Py_DECREF(iterable); + SET_TOP(NULL); + PyErr_SetString(PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "from a generator"); + goto error; + } + } + else if (!PyGen_CheckExact(iterable)) { /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -2949,7 +2945,6 @@ if (iter == NULL) goto error; } - PREDICT(FOR_ITER); DISPATCH(); } @@ -3942,7 +3937,11 @@ /* Create a new generator that owns the ready to run frame * and return that as the value. */ - gen = PyGen_NewWithQualName(f, name, qualname); + if (co->co_flags & CO_COROUTINE) { + gen = PyCoro_NewWithQualName(f, name, qualname); + } else { + gen = PyGen_NewWithQualName(f, name, qualname); + } if (gen == NULL) return NULL; diff -r 38f75ebfe4cd Python/compile.c --- a/Python/compile.c Tue Jun 09 18:42:36 2015 +0300 +++ b/Python/compile.c Tue Jun 09 12:35:35 2015 -0400 @@ -1064,6 +1064,8 @@ return 0; case GET_ANEXT: return 1; + case GET_YIELD_FROM_ITER: + return 0; default: return PY_INVALID_STACK_EFFECT; } @@ -3850,7 +3852,7 @@ return compiler_error(c, "'yield from' inside async function"); VISIT(c, expr, e->v.YieldFrom.value); - ADDOP(c, GET_ITER); + ADDOP(c, GET_YIELD_FROM_ITER); ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); break; diff -r 38f75ebfe4cd Python/opcode_targets.h --- a/Python/opcode_targets.h Tue Jun 09 18:42:36 2015 +0300 +++ b/Python/opcode_targets.h Tue Jun 09 12:35:35 2015 -0400 @@ -68,7 +68,7 @@ &&TARGET_BINARY_OR, &&TARGET_INPLACE_POWER, &&TARGET_GET_ITER, - &&_unknown_opcode, + &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_YIELD_FROM,