11"""
2- To ensure compatibility from Python ``2.6 `` - ``3.3 ``, a module has been
2+ To ensure compatibility from Python ``2.7 `` - ``3.x ``, a module has been
33created. Clearly there is huge need to use conforming syntax.
44"""
55import sys
6- import imp
76import os
87import re
98import pkgutil
109import warnings
10+ import inspect
1111try :
1212 import importlib
1313except ImportError :
1414 pass
1515
16- # Cannot use sys.version.major and minor names, because in Python 2.6 it's not
17- # a namedtuple.
1816is_py3 = sys .version_info [0 ] >= 3
1917is_py33 = is_py3 and sys .version_info [1 ] >= 3
2018is_py34 = is_py3 and sys .version_info [1 ] >= 4
2119is_py35 = is_py3 and sys .version_info [1 ] >= 5
22- is_py26 = not is_py3 and sys .version_info [1 ] < 7
2320py_version = int (str (sys .version_info [0 ]) + str (sys .version_info [1 ]))
2421
2522
@@ -35,7 +32,7 @@ def close(self):
3532 del self .loader
3633
3734
38- def find_module_py34 (string , path = None , fullname = None ):
35+ def find_module_py34 (string , path = None , full_name = None ):
3936 implicit_namespace_pkg = False
4037 spec = None
4138 loader = None
@@ -47,16 +44,17 @@ def find_module_py34(string, path=None, fullname=None):
4744
4845 # We try to disambiguate implicit namespace pkgs with non implicit namespace pkgs
4946 if implicit_namespace_pkg :
50- fullname = string if not path else fullname
51- implicit_ns_info = ImplicitNSInfo (fullname , spec .submodule_search_locations ._path )
47+ full_name = string if not path else full_name
48+ implicit_ns_info = ImplicitNSInfo (full_name , spec .submodule_search_locations ._path )
5249 return None , implicit_ns_info , False
5350
5451 # we have found the tail end of the dotted path
5552 if hasattr (spec , 'loader' ):
5653 loader = spec .loader
5754 return find_module_py33 (string , path , loader )
5855
59- def find_module_py33 (string , path = None , loader = None , fullname = None ):
56+
57+ def find_module_py33 (string , path = None , loader = None , full_name = None ):
6058 loader = loader or importlib .machinery .PathFinder .find_module (string , path )
6159
6260 if loader is None and path is None : # Fallback to find builtins
@@ -74,7 +72,7 @@ def find_module_py33(string, path=None, loader=None, fullname=None):
7472 raise ImportError ("Originally " + repr (e ))
7573
7674 if loader is None :
77- raise ImportError ("Couldn't find a loader for {0 }" .format (string ))
75+ raise ImportError ("Couldn't find a loader for {}" .format (string ))
7876
7977 try :
8078 is_package = loader .is_package (string )
@@ -109,7 +107,10 @@ def find_module_py33(string, path=None, loader=None, fullname=None):
109107 return module_file , module_path , is_package
110108
111109
112- def find_module_pre_py33 (string , path = None , fullname = None ):
110+ def find_module_pre_py33 (string , path = None , full_name = None ):
111+ # This import is here, because in other places it will raise a
112+ # DeprecationWarning.
113+ import imp
113114 try :
114115 module_file , module_path , description = imp .find_module (string , path )
115116 module_type = description [2 ]
@@ -127,29 +128,22 @@ def find_module_pre_py33(string, path=None, fullname=None):
127128 if loader :
128129 is_package = loader .is_package (string )
129130 is_archive = hasattr (loader , 'archive' )
130- try :
131- module_path = loader .get_filename (string )
132- except AttributeError :
133- # fallback for py26
134- try :
135- module_path = loader ._get_filename (string )
136- except AttributeError :
137- continue
131+ module_path = loader .get_filename (string )
138132 if is_package :
139133 module_path = os .path .dirname (module_path )
140134 if is_archive :
141135 module_path = loader .archive
142136 file = None
143137 if not is_package or is_archive :
144138 file = DummyFile (loader , string )
145- return ( file , module_path , is_package )
139+ return file , module_path , is_package
146140 except ImportError :
147141 pass
148- raise ImportError ("No module named {0 }" .format (string ))
142+ raise ImportError ("No module named {}" .format (string ))
149143
150144
151145find_module = find_module_py33 if is_py33 else find_module_pre_py33
152- find_module = find_module_py34 if is_py34 else find_module
146+ find_module = find_module_py34 if is_py34 else find_module
153147find_module .__doc__ = """
154148Provides information about a module.
155149
@@ -161,12 +155,80 @@ def find_module_pre_py33(string, path=None, fullname=None):
161155"""
162156
163157
158+ def _iter_modules (paths , prefix = '' ):
159+ # Copy of pkgutil.iter_modules adapted to work with namespaces
160+
161+ for path in paths :
162+ importer = pkgutil .get_importer (path )
163+
164+ if not isinstance (importer , importlib .machinery .FileFinder ):
165+ # We're only modifying the case for FileFinder. All the other cases
166+ # still need to be checked (like zip-importing). Do this by just
167+ # calling the pkgutil version.
168+ for mod_info in pkgutil .iter_modules ([path ], prefix ):
169+ yield mod_info
170+ continue
171+
172+ # START COPY OF pkutils._iter_file_finder_modules.
173+ if importer .path is None or not os .path .isdir (importer .path ):
174+ return
175+
176+ yielded = {}
177+
178+ try :
179+ filenames = os .listdir (importer .path )
180+ except OSError :
181+ # ignore unreadable directories like import does
182+ filenames = []
183+ filenames .sort () # handle packages before same-named modules
184+
185+ for fn in filenames :
186+ modname = inspect .getmodulename (fn )
187+ if modname == '__init__' or modname in yielded :
188+ continue
189+
190+ # jedi addition: Avoid traversing special directories
191+ if fn .startswith ('.' ) or fn == '__pycache__' :
192+ continue
193+
194+ path = os .path .join (importer .path , fn )
195+ ispkg = False
196+
197+ if not modname and os .path .isdir (path ) and '.' not in fn :
198+ modname = fn
199+ # A few jedi modifications: Don't check if there's an
200+ # __init__.py
201+ try :
202+ os .listdir (path )
203+ except OSError :
204+ # ignore unreadable directories like import does
205+ continue
206+ ispkg = True
207+
208+ if modname and '.' not in modname :
209+ yielded [modname ] = 1
210+ yield importer , prefix + modname , ispkg
211+ # END COPY
212+
213+ iter_modules = _iter_modules if py_version >= 34 else pkgutil .iter_modules
214+
215+
164216class ImplicitNSInfo (object ):
165217 """Stores information returned from an implicit namespace spec"""
166218 def __init__ (self , name , paths ):
167219 self .name = name
168220 self .paths = paths
169221
222+
223+ if is_py3 :
224+ all_suffixes = importlib .machinery .all_suffixes
225+ else :
226+ def all_suffixes ():
227+ # Is deprecated and raises a warning in Python 3.6.
228+ import imp
229+ return [suffix for suffix , _ , _ in imp .get_suffixes ()]
230+
231+
170232# unicode function
171233try :
172234 unicode = unicode
@@ -219,19 +281,37 @@ def use_metaclass(meta, *bases):
219281 encoding = 'ascii'
220282
221283
222- def u (string ):
284+ def u (string , errors = 'strict' ):
223285 """Cast to unicode DAMMIT!
224286 Written because Python2 repr always implicitly casts to a string, so we
225287 have to cast back to a unicode (and we now that we always deal with valid
226288 unicode, because we check that in the beginning).
227289 """
228- if is_py3 :
229- return str (string )
230-
231- if not isinstance (string , unicode ):
232- return unicode (str (string ), 'UTF-8' )
290+ if isinstance (string , bytes ):
291+ return unicode (string , encoding = 'UTF-8' , errors = errors )
233292 return string
234293
294+
295+ def cast_path (obj ):
296+ """
297+ Take a bytes or str path and cast it to unicode.
298+
299+ Apparently it is perfectly fine to pass both byte and unicode objects into
300+ the sys.path. This probably means that byte paths are normal at other
301+ places as well.
302+
303+ Since this just really complicates everything and Python 2.7 will be EOL
304+ soon anyway, just go with always strings.
305+ """
306+ return u (obj , errors = 'replace' )
307+
308+
309+ def force_unicode (obj ):
310+ # Intentionally don't mix those two up, because those two code paths might
311+ # be different in the future (maybe windows?).
312+ return cast_path (obj )
313+
314+
235315try :
236316 import builtins # module name in python 3
237317except ImportError :
@@ -242,11 +322,6 @@ def u(string):
242322
243323
244324def literal_eval (string ):
245- # py3.0, py3.1 and py32 don't support unicode literals. Support those, I
246- # don't want to write two versions of the tokenizer.
247- if is_py3 and sys .version_info .minor < 3 :
248- if re .match ('[uU][\' "]' , string ):
249- string = string [1 :]
250325 return ast .literal_eval (string )
251326
252327
@@ -260,6 +335,11 @@ def literal_eval(string):
260335except NameError :
261336 FileNotFoundError = IOError
262337
338+ try :
339+ NotADirectoryError = NotADirectoryError
340+ except NameError :
341+ NotADirectoryError = IOError
342+
263343
264344def no_unicode_pprint (dct ):
265345 """
@@ -273,6 +353,13 @@ def no_unicode_pprint(dct):
273353 print (re .sub ("u'" , "'" , s ))
274354
275355
356+ def print_to_stderr (string ):
357+ if is_py3 :
358+ eval ("print(string, file=sys.stderr)" )
359+ else :
360+ print >> sys .stderr , string
361+
362+
276363def utf8_repr (func ):
277364 """
278365 ``__repr__`` methods in Python 2 don't allow unicode objects to be
@@ -289,3 +376,95 @@ def wrapper(self):
289376 return func
290377 else :
291378 return wrapper
379+
380+
381+ if is_py3 :
382+ import queue
383+ else :
384+ import Queue as queue
385+
386+
387+ import pickle
388+ if sys .version_info [:2 ] == (3 , 3 ):
389+ """
390+ Monkeypatch the unpickler in Python 3.3. This is needed, because the
391+ argument `encoding='bytes'` is not supported in 3.3, but badly needed to
392+ communicate with Python 2.
393+ """
394+
395+ class NewUnpickler (pickle ._Unpickler ):
396+ dispatch = dict (pickle ._Unpickler .dispatch )
397+
398+ def _decode_string (self , value ):
399+ # Used to allow strings from Python 2 to be decoded either as
400+ # bytes or Unicode strings. This should be used only with the
401+ # STRING, BINSTRING and SHORT_BINSTRING opcodes.
402+ if self .encoding == "bytes" :
403+ return value
404+ else :
405+ return value .decode (self .encoding , self .errors )
406+
407+ def load_string (self ):
408+ data = self .readline ()[:- 1 ]
409+ # Strip outermost quotes
410+ if len (data ) >= 2 and data [0 ] == data [- 1 ] and data [0 ] in b'"\' ' :
411+ data = data [1 :- 1 ]
412+ else :
413+ raise pickle .UnpicklingError ("the STRING opcode argument must be quoted" )
414+ self .append (self ._decode_string (pickle .codecs .escape_decode (data )[0 ]))
415+ dispatch [pickle .STRING [0 ]] = load_string
416+
417+ def load_binstring (self ):
418+ # Deprecated BINSTRING uses signed 32-bit length
419+ len , = pickle .struct .unpack ('<i' , self .read (4 ))
420+ if len < 0 :
421+ raise pickle .UnpicklingError ("BINSTRING pickle has negative byte count" )
422+ data = self .read (len )
423+ self .append (self ._decode_string (data ))
424+ dispatch [pickle .BINSTRING [0 ]] = load_binstring
425+
426+ def load_short_binstring (self ):
427+ len = self .read (1 )[0 ]
428+ data = self .read (len )
429+ self .append (self ._decode_string (data ))
430+ dispatch [pickle .SHORT_BINSTRING [0 ]] = load_short_binstring
431+
432+ def load (file , fix_imports = True , encoding = "ASCII" , errors = "strict" ):
433+ return NewUnpickler (file , fix_imports = fix_imports ,
434+ encoding = encoding , errors = errors ).load ()
435+
436+ def loads (s , fix_imports = True , encoding = "ASCII" , errors = "strict" ):
437+ if isinstance (s , str ):
438+ raise TypeError ("Can't load pickle from unicode string" )
439+ file = pickle .io .BytesIO (s )
440+ return NewUnpickler (file , fix_imports = fix_imports ,
441+ encoding = encoding , errors = errors ).load ()
442+
443+ pickle .Unpickler = NewUnpickler
444+ pickle .load = load
445+ pickle .loads = loads
446+
447+
448+ _PICKLE_PROTOCOL = 2
449+
450+
451+ def pickle_load (file ):
452+ if is_py3 :
453+ return pickle .load (file , encoding = 'bytes' )
454+ else :
455+ return pickle .load (file )
456+
457+
458+ def pickle_dump (data , file ):
459+ pickle .dump (data , file , protocol = _PICKLE_PROTOCOL )
460+
461+
462+ try :
463+ from inspect import Parameter
464+ except ImportError :
465+ class Parameter (object ):
466+ POSITIONAL_ONLY = object ()
467+ POSITIONAL_OR_KEYWORD = object ()
468+ VAR_POSITIONAL = object ()
469+ KEYWORD_ONLY = object ()
470+ VAR_KEYWORD = object ()
0 commit comments