Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 41 additions & 11 deletions Lib/pickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

Misc variables:

__version__
format_version
compatible_formats

Expand Down Expand Up @@ -190,6 +189,11 @@ def __init__(self, value):
__all__.extend(x for x in dir() if x.isupper() and not x.startswith('_'))


# Data larger than this will be read in chunks, to prevent extreme
# overallocation.
_MIN_READ_BUF_SIZE = (1 << 20)


class _Framer:

_FRAME_SIZE_MIN = 4
Expand Down Expand Up @@ -288,7 +292,7 @@ def read(self, n):
"pickle exhausted before end of frame")
return data
else:
return self.file_read(n)
return self._chunked_file_read(n)

def readline(self):
if self.current_frame:
Expand All @@ -303,11 +307,23 @@ def readline(self):
else:
return self.file_readline()

def _chunked_file_read(self, size):
cursize = min(size, _MIN_READ_BUF_SIZE)
b = self.file_read(cursize)
while cursize < size and len(b) == cursize:
delta = min(cursize, size - cursize)
b += self.file_read(delta)
cursize += delta
return b

def load_frame(self, frame_size):
if self.current_frame and self.current_frame.read() != b'':
raise UnpicklingError(
"beginning of a new frame before end of current frame")
self.current_frame = io.BytesIO(self.file_read(frame_size))
data = self._chunked_file_read(frame_size)
if len(data) < frame_size:
raise EOFError
self.current_frame = io.BytesIO(data)


# Tools used for pickling.
Expand Down Expand Up @@ -1147,13 +1163,23 @@ def save_frozenset(self, obj):

def save_global(self, obj, name=None):
write = self.write
memo = self.memo

if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__

if '.__' in name:
# Mangle names of private attributes.
dotted_path = name.split('.')
for i, subpath in enumerate(dotted_path):
if i and subpath.startswith('__') and not subpath.endswith('__'):
prev = prev.lstrip('_')
if prev:
dotted_path[i] = f"_{prev.lstrip('_')}{subpath}"
prev = subpath
name = '.'.join(dotted_path)

module_name = whichmodule(obj, name)
if self.proto >= 2:
code = _extension_registry.get((module_name, name), _NoValue)
Expand Down Expand Up @@ -1491,12 +1517,17 @@ def load_binbytes8(self):
dispatch[BINBYTES8[0]] = load_binbytes8

def load_bytearray8(self):
len, = unpack('<Q', self.read(8))
if len > maxsize:
size, = unpack('<Q', self.read(8))
if size > maxsize:
raise UnpicklingError("BYTEARRAY8 exceeds system's maximum size "
"of %d bytes" % maxsize)
b = bytearray(len)
self.readinto(b)
cursize = min(size, _MIN_READ_BUF_SIZE)
b = bytearray(cursize)
if self.readinto(b) == cursize:
while cursize < size and len(b) == cursize:
delta = min(cursize, size - cursize)
b += self.read(delta)
cursize += delta
self.append(b)
dispatch[BYTEARRAY8[0]] = load_bytearray8

Expand Down Expand Up @@ -1729,7 +1760,7 @@ def load_binget(self):
i = self.read(1)[0]
try:
self.append(self.memo[i])
except KeyError as exc:
except KeyError:
msg = f'Memo value not found at index {i}'
raise UnpicklingError(msg) from None
dispatch[BINGET[0]] = load_binget
Expand All @@ -1738,7 +1769,7 @@ def load_long_binget(self):
i, = unpack('<I', self.read(4))
try:
self.append(self.memo[i])
except KeyError as exc:
except KeyError:
msg = f'Memo value not found at index {i}'
raise UnpicklingError(msg) from None
dispatch[LONG_BINGET[0]] = load_long_binget
Expand Down Expand Up @@ -1906,7 +1937,6 @@ def _main(args=None):
import pprint
parser = argparse.ArgumentParser(
description='display contents of the pickle files',
color=True,
)
parser.add_argument(
'pickle_file',
Expand Down
83 changes: 65 additions & 18 deletions Lib/pickletools.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import re
import sys

lazy from _colorize import decolor, get_theme

__all__ = ['dis', 'genops', 'optimize']

bytes_types = pickle.bytes_types
Expand Down Expand Up @@ -2209,6 +2211,32 @@ def __init__(self, name, code, arg,
name2i[d.name] = i
code2i[d.code] = i

# Group opcode names into categories for colourised CLI output.
_opcode_categories = frozendict(
op_call=frozenset({
"BUILD", "EXT1", "EXT2", "EXT4", "GLOBAL", "INST", "NEWOBJ",
"NEWOBJ_EX", "OBJ", "REDUCE", "STACK_GLOBAL",
}),
op_container=frozenset({
"ADDITEMS", "APPEND", "APPENDS", "DICT", "EMPTY_DICT", "EMPTY_LIST",
"EMPTY_SET", "EMPTY_TUPLE", "FROZENSET", "LIST", "SETITEM",
"SETITEMS", "TUPLE", "TUPLE1", "TUPLE2", "TUPLE3",
}),
op_memo=frozenset({
"BINGET", "BINPUT", "GET", "LONG_BINGET", "LONG_BINPUT", "MEMOIZE",
"PUT",
}),
op_meta=frozenset({"BINPERSID", "FRAME", "MARK", "PERSID", "PROTO"}),
op_stack=frozenset({"DUP", "POP", "POP_MARK", "STOP"}),
)
_opcode_color_attr = frozendict({
name: attr
for attr, names in _opcode_categories.items()
for name in names
})
assert _opcode_color_attr.keys() <= name2i.keys(), (
f"unknown opcodes: {_opcode_color_attr.keys() - name2i.keys()}"
)
del name2i, code2i, i, d

##############################################################################
Expand Down Expand Up @@ -2443,13 +2471,19 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0):
indentchunk = ' ' * indentlevel
errormsg = None
annocol = annotate # column hint for annotations
t = get_theme(tty_file=out).pickletools
for opcode, arg, pos in genops(pickle):
if pos is not None:
print("%5d:" % pos, end=' ', file=out)
print(f"{t.position}{pos:5d}:{t.reset}", end=' ', file=out)

line = "%-4s %s%s" % (repr(opcode.code)[1:-1],
indentchunk * len(markstack),
opcode.name)
attr = _opcode_color_attr.get(opcode.name)
opcode_color = getattr(t, attr) if attr else ""
opcode_reset = t.reset if attr else ""
line = (
f"{t.opcode_code}{repr(opcode.code)[1:-1]:<4}{t.reset} "
f"{indentchunk * len(markstack)}"
f"{opcode_color}{opcode.name}{opcode_reset}"
)

maxproto = max(maxproto, opcode.proto)
before = opcode.stack_before # don't mutate
Expand Down Expand Up @@ -2510,18 +2544,26 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0):
line += ' ' * (10 - len(opcode.name))
if arg is not None:
if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"):
line += ' ' + ascii(arg)
arg_text = ascii(arg)
else:
line += ' ' + repr(arg)
arg_text = repr(arg)
arg_color = (
t.arg_number
if isinstance(arg, (int, float))
else t.arg_string
)
line += f" {arg_color}{arg_text}{t.reset}"
if markmsg:
line += ' ' + markmsg
line += f" {t.mark}{markmsg}{t.reset}"
if annotate:
line += ' ' * (annocol - len(line))
visible_len = len(decolor(line))
line += ' ' * (annocol - visible_len)
# make a mild effort to align annotations
annocol = len(line)
annocol = max(visible_len, annocol)
if annocol > 50:
annocol = annotate
line += ' ' + opcode.doc.split('\n', 1)[0]
doc = opcode.doc.split('\n', 1)[0]
line += f" {t.annotation}{doc}{t.reset}"
print(line, file=out)

if errormsg:
Expand All @@ -2541,7 +2583,11 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0):

stack.extend(after)

print("highest protocol among opcodes =", maxproto, file=out)
print(
"highest protocol among opcodes =",
f"{t.proto}{maxproto}{t.reset}",
file=out,
)
if stack:
raise ValueError("stack not empty after STOP: %r" % stack)

Expand Down Expand Up @@ -2839,12 +2885,9 @@ def __init__(self, value):
}


if __name__ == "__main__":
def _main(args=None):
import argparse
parser = argparse.ArgumentParser(
description='disassemble one or more pickle files',
color=True,
)
parser = argparse.ArgumentParser(description='disassemble one or more pickle files')
parser.add_argument(
'pickle_file',
nargs='+', help='the pickle file')
Expand All @@ -2856,15 +2899,15 @@ def __init__(self, value):
help='preserve memo between disassemblies')
parser.add_argument(
'-l', '--indentlevel', default=4, type=int,
help='the number of blanks by which to indent a new MARK level')
help='the number of blanks by which to indent a new `MARK` level')
parser.add_argument(
'-a', '--annotate', action='store_true',
help='annotate each line with a short opcode description')
parser.add_argument(
'-p', '--preamble', default="==> {name} <==",
help='if more than one pickle file is specified, print this before'
' each disassembly')
args = parser.parse_args()
args = parser.parse_args(args)
annotate = 30 if args.annotate else 0
memo = {} if args.memo else None
if args.output is None:
Expand All @@ -2885,3 +2928,7 @@ def __init__(self, value):
finally:
if output is not sys.stdout:
output.close()


if __name__ == "__main__":
_main()
56 changes: 56 additions & 0 deletions Lib/test/picklecommon.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,17 @@ class MyFrozenSet(frozenset):
MyStr, MyUnicode,
MyTuple, MyList, MyDict, MySet, MyFrozenSet]

try:
frozendict
except NameError:
# Python 3.14 and older
pass
else:
class MyFrozenDict(dict):
sample = frozendict({"a": 1, "b": 2})
myclasses.append(MyFrozenDict)


# For test_newobj_overridden_new
class MyIntWithNew(int):
def __new__(cls, value):
Expand Down Expand Up @@ -388,3 +399,48 @@ def pie(self):
class Subclass(tuple):
class Nested(str):
pass

# For test_private_methods
class PrivateMethods:
def __init__(self, value):
self.value = value

def __private_method(self):
return self.value

def get_method(self):
return self.__private_method

@classmethod
def get_unbound_method(cls):
return cls.__private_method

@classmethod
def __private_classmethod(cls):
return 43

@classmethod
def get_classmethod(cls):
return cls.__private_classmethod

@staticmethod
def __private_staticmethod():
return 44

@classmethod
def get_staticmethod(cls):
return cls.__private_staticmethod

# For test_private_nested_classes
class PrivateNestedClasses:
@classmethod
def get_nested(cls):
return cls.__Nested

class __Nested:
@classmethod
def get_nested2(cls):
return cls.__Nested2

class __Nested2:
pass
Loading
Loading