diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -191,6 +191,97 @@ def _new_module(name):
return type(sys)(name)
+# Module-level locking ########################################################
+
+_module_locks = {}
+_blocking_on = {}
+
+class _ModuleLock:
+
+ def __init__(self, name):
+ self.lock = _thread.allocate_lock()
+ self.wakeup = _thread.allocate_lock()
+ self.name = name
+ self.owner = None
+ self.count = 0
+ self.waiters = 0
+
+ def has_deadlock(self):
+ # Deadlock avoidance for concurrent circular imports.
+ me = _thread.get_ident()
+ tid = self.owner
+ while True:
+ lock = _blocking_on.get(tid)
+ if lock is None:
+ return False
+ tid = lock.owner
+ if tid == me:
+ return True
+
+ def acquire(self, avoid_deadlocks=False):
+ tid = _thread.get_ident()
+ _blocking_on[tid] = self
+ try:
+ while True:
+ with self.lock:
+ if self.count == 0 or self.owner == tid:
+ self.owner = tid
+ self.count += 1
+ return True
+ if avoid_deadlocks and self.has_deadlock():
+ return False
+ if self.wakeup.acquire(False):
+ self.waiters += 1
+ self.wakeup.acquire()
+ self.wakeup.release()
+ finally:
+ del _blocking_on[tid]
+
+ def release(self):
+ tid = _thread.get_ident()
+ with self.lock:
+ assert self.owner == tid
+ assert self.count > 0
+ self.count -= 1
+ if self.count == 0:
+ self.owner = None
+ if self.waiters:
+ self.waiters -= 1
+ self.wakeup.release()
+
+ def __repr__(self):
+ return "_ModuleLock(%r) at %d" % (self.name, id(self))
+
+
+def _get_module_lock(name):
+ # Should only be called with the import lock taken.
+ lock = None
+ if name in _module_locks:
+ lock = _module_locks[name]()
+ if lock is None:
+ lock = _ModuleLock(name)
+ def cb(_):
+ del _module_locks[name]
+ _module_locks[name] = _weakref.ref(lock, cb)
+ return lock
+
+def _get_module_lock(name):
+ # Should only be called with the import lock taken.
+ if name in _module_locks:
+ lock = _module_locks[name]
+ else:
+ lock = _ModuleLock(name)
+ _module_locks[name] = lock
+ return lock
+
+def _lock_unlock_module(name):
+ # Should only be called with the import lock taken. Releases it.
+ lock = _get_module_lock(name)
+ _imp.release_lock()
+ if lock.acquire(avoid_deadlocks=True):
+ lock.release()
+
+
# Finder/loader utility code ##################################################
PYCACHE = '__pycache__'
@@ -290,12 +381,15 @@ def module_for_loader(fxn):
else:
module.__package__ = fullname.rpartition('.')[0]
try:
+ module.__initializing__ = True
# If __package__ was not set above, __import__() will do it later.
return fxn(self, module, *args, **kwargs)
except:
if not is_reload:
del sys.modules[fullname]
raise
+ finally:
+ del module.__initializing__
_wrap(module_for_loader_wrapper, fxn)
return module_for_loader_wrapper
@@ -960,7 +1054,8 @@ def _find_module(name, path):
if not sys.meta_path:
_warnings.warn('sys.meta_path is empty', ImportWarning)
for finder in sys.meta_path:
- loader = finder.find_module(name, path)
+ with _ImportLockContext():
+ loader = finder.find_module(name, path)
if loader is not None:
# The parent import may have already imported this module.
if name not in sys.modules:
@@ -990,8 +1085,7 @@ def _sanity_check(name, package, level):
_ERR_MSG = 'No module named {!r}'
-def _find_and_load(name, import_):
- """Find and load the module."""
+def _find_and_load_unlocked(name, import_):
path = None
parent = name.rpartition('.')[0]
if parent:
@@ -1037,6 +1131,19 @@ def _find_and_load(name, import_):
return module
+def _find_and_load(name, import_):
+ """Find and load the module, and release the import lock."""
+ try:
+ lock = _get_module_lock(name)
+ finally:
+ _imp.release_lock()
+ lock.acquire()
+ try:
+ return _find_and_load_unlocked(name, import_)
+ finally:
+ lock.release()
+
+
def _gcd_import(name, package=None, level=0):
"""Import and return the module based on its name, the package the call is
being made from, and the level adjustment.
@@ -1049,17 +1156,17 @@ def _gcd_import(name, package=None, leve
_sanity_check(name, package, level)
if level > 0:
name = _resolve_name(name, package, level)
- with _ImportLockContext():
- try:
- module = sys.modules[name]
- if module is None:
- message = ("import of {} halted; "
- "None in sys.modules".format(name))
- raise ImportError(message, name=name)
- return module
- except KeyError:
- pass # Don't want to chain the exception
+ _imp.acquire_lock()
+ if name not in sys.modules:
return _find_and_load(name, _gcd_import)
+ module = sys.modules[name]
+ if module is None:
+ _imp.release_lock()
+ message = ("import of {} halted; "
+ "None in sys.modules".format(name))
+ raise ImportError(message, name=name)
+ _lock_unlock_module(name)
+ return module
def _handle_fromlist(module, fromlist, import_):
@@ -1177,7 +1284,17 @@ def _setup(sys_module, _imp_module):
continue
else:
raise ImportError('importlib requires posix or nt')
+
+ try:
+ thread_module = BuiltinImporter.load_module('_thread')
+ except ImportError:
+ # Python was built without threads
+ thread_module = None
+ weakref_module = BuiltinImporter.load_module('_weakref')
+
setattr(self_module, '_os', os_module)
+ setattr(self_module, '_thread', thread_module)
+ setattr(self_module, '_weakref', weakref_module)
setattr(self_module, 'path_sep', path_sep)
setattr(self_module, 'path_separators', set(path_separators))
# Constants
diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py
--- a/Lib/test/test_threaded_import.py
+++ b/Lib/test/test_threaded_import.py
@@ -12,7 +12,7 @@ import time
import shutil
import unittest
from test.support import (
- verbose, import_module, run_unittest, TESTFN, reap_threads)
+ verbose, import_module, run_unittest, TESTFN, reap_threads, forget)
threading = import_module('threading')
def task(N, done, done_tasks, errors):
@@ -187,7 +187,7 @@ class ThreadedImportTests(unittest.TestC
contents = contents % {'delay': delay}
with open(os.path.join(TESTFN, name + ".py"), "wb") as f:
f.write(contents.encode('utf-8'))
- self.addCleanup(sys.modules.pop, name, None)
+ self.addCleanup(forget, name)
results = []
def import_ab():
@@ -204,6 +204,21 @@ class ThreadedImportTests(unittest.TestC
t2.join()
self.assertEqual(set(results), {'a', 'b'})
+ def test_side_effect_import(self):
+ code = """if 1:
+ import threading
+ def target():
+ import random
+ t = threading.Thread(target=target)
+ t.start()
+ t.join()"""
+ sys.path.insert(0, os.curdir)
+ self.addCleanup(sys.path.remove, os.curdir)
+ with open(TESTFN + ".py", "wb") as f:
+ f.write(code.encode('utf-8'))
+ self.addCleanup(forget, TESTFN)
+ __import__(TESTFN)
+
@reap_threads
def test_main():
diff --git a/Lib/token.py b/Lib/token.py
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -70,7 +70,7 @@ NT_OFFSET = 256
tok_name = {value: name
for name, value in globals().items()
- if isinstance(value, int)}
+ if isinstance(value, int) and not name.startswith('_')}
__all__.extend(tok_name.values())
def ISTERMINAL(x):
diff --git a/Python/import.c b/Python/import.c
--- a/Python/import.c
+++ b/Python/import.c
@@ -1488,11 +1488,13 @@ PyImport_ImportModuleLevelObject(PyObjec
int level)
{
_Py_IDENTIFIER(__import__);
+ _Py_IDENTIFIER(__initializing__);
_Py_IDENTIFIER(__package__);
_Py_IDENTIFIER(__path__);
_Py_IDENTIFIER(__name__);
_Py_IDENTIFIER(_find_and_load);
_Py_IDENTIFIER(_handle_fromlist);
+ _Py_IDENTIFIER(_lock_unlock_module);
_Py_static_string(single_dot, ".");
PyObject *abs_name = NULL;
PyObject *builtins_import = NULL;
@@ -1675,16 +1677,48 @@ PyImport_ImportModuleLevelObject(PyObjec
goto error_with_unlock;
}
else if (mod != NULL) {
+ PyObject *value;
+ int initializing = 0;
+
Py_INCREF(mod);
+ /* Only call _bootstrap._lock_unlock_module() if __initializing__ is true. */
+ value = _PyObject_GetAttrId(mod, &PyId___initializing__);
+ if (value == NULL)
+ PyErr_Clear();
+ else {
+ initializing = PyObject_IsTrue(value);
+ Py_DECREF(value);
+ if (initializing == -1)
+ PyErr_Clear();
+ }
+ if (initializing > 0) {
+ /* _bootstrap._lock_unlock_module() releases the import lock */
+ value = _PyObject_CallMethodObjIdArgs(interp->importlib,
+ &PyId__lock_unlock_module, abs_name,
+ NULL);
+ if (value == NULL)
+ goto error;
+ Py_DECREF(value);
+ }
+ else {
+#ifdef WITH_THREAD
+ if (_PyImport_ReleaseLock() < 0) {
+ PyErr_SetString(PyExc_RuntimeError, "not holding the import lock");
+ goto error;
+ }
+#endif
+ }
}
else {
+ /* _bootstrap._find_and_load() releases the import lock */
mod = _PyObject_CallMethodObjIdArgs(interp->importlib,
&PyId__find_and_load, abs_name,
builtins_import, NULL);
if (mod == NULL) {
- goto error_with_unlock;
+ goto error;
}
}
+ /* From now on we don't hold the import lock anymore. */
if (PyObject_Not(fromlist)) {
if (level == 0 || PyUnicode_GET_LENGTH(name) > 0) {
@@ -1693,12 +1727,12 @@ PyImport_ImportModuleLevelObject(PyObjec
PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot);
if (borrowed_dot == NULL) {
- goto error_with_unlock;
+ goto error;
}
partition = PyUnicode_Partition(name, borrowed_dot);
if (partition == NULL) {
- goto error_with_unlock;
+ goto error;
}
front = PyTuple_GET_ITEM(partition, 0);
@@ -1724,7 +1758,7 @@ PyImport_ImportModuleLevelObject(PyObjec
abs_name_len - cut_off);
Py_DECREF(front);
if (to_return == NULL) {
- goto error_with_unlock;
+ goto error;
}
final_mod = PyDict_GetItem(interp->modules, to_return);
@@ -1750,6 +1784,8 @@ PyImport_ImportModuleLevelObject(PyObjec
fromlist, builtins_import,
NULL);
}
+ goto error;
+
error_with_unlock:
#ifdef WITH_THREAD
if (_PyImport_ReleaseLock() < 0) {
diff --git a/Python/importlib.h b/Python/importlib.h
index e9f5faf2fa9b7e239563aff1db46d6e1d70fcb88..83c792a2e46befc55b6152c3dde5606e38e6a512
GIT binary patch
literal 184458
zc%1Eh$&xh5ah}e6iaEndLPq2+nGSjYn(;1^!9WTkVIo1=2s}VCy}K$SeEHnNBQmRJ
zKr|>4!@s+$GGq1d|(e`HLU^%Rm4AAOHQ|{^obT`SI7k|IL5@;jjPj!(aX3_kZ_~
zzx(C?`0+Qt`QN|!)o=gdpZ@X3-~at@|N1}t(;t5L#V`NjpMUqie*3q-dq4R%fA{Mj
zfB4m3{oz-C`73?szy7x${__w2=P!PE|KEAOrq2Ji@^5+S{l`L|rnJ76?KRgwruyMI
z>9f~oX*%ksi5^<$=X(8|Grv?`$n@PtKX?1gOdlUSl `#M-
zsr!$yu`^wm<$Im0F SFM!jn}^DXq?c>Z
zy{W#p%=L<`WMf(+37t7SMDnA^I^-BorHaH?+F0-jYVm8mRgpD1_Jf|@nIlYEaFrS@
z(Rc0*YN_PyLKHpM>94Wz{uViNfm~J3+ejV>Bw`iMvQdsY=m?*haT7<02*V*UlGTV?
zG2#YM;d2K)cy!f`mn?G0Su%t?3Q9EzQ|%yva6^}Tr^aWTfoEB&Nxl;t<+Je{R)ynta4Sle*VQM$31-$O+Puhw>8-
zi8~+0YVDLccITt6>iCc}=R+~d;E?=DVMZAb5TyWr@p|Ws;wQT)zDz*GJ^UlCnrEja
z7?Uj3;M7=va&&4S#l@T&rlP)JDeruKYD_fg{i$&nkr;|D6k5TtsHQ;`A#s|kTmmqm
zOTiovLF*rzj-Z0}Ps#VaoJvm#sGg&;S$Up=Gvba1OUI8OQam12
RDj*22poN?J-(?
`i>cDaEhR4)aqQF`dw>YYYCc
z|4wv17H#T(`wRm-1;y%OfL=?U>3H6Z!{tE*Jgjq}$%R^JFq5fN1O|yrfC8CtaDXcs
zE=U8MX3-fi;fnQ>Q`yY4I~0^%m8B!U(riLauSU}G`(~nHo*VGRah_XKN|-n^f9LYt
z8tmI<(qPIKY}Uh+t{+4zgGKr2#507%toR(LL@hHRM-i|H>-bcsEV~m=2CR5M
z3Mqj1iXCSnzL-pCc+s3~f(?c}WcYi!nYFN`v9`nYqjG{VJ1=Ca!g^6FmX}q$ufy6H
z6CaMru-q#i@CuA`(SgfZKe8H!QpfUAw<|