Skip to content

Commit ea780ac

Browse files
author
James William Pye
committed
Add protocol optimizations; pack_tuple_data and cat_messages.
 * Provides a more efficient means to construct the send buffer from the out-going messsages. * pack_tuple_data is used by Tuple.serialize to quickly render the serialized form. * add ulong and ushort optimizations. * minor refactoring to parse_tuple_message. Notably, the new cat_messages function will take note of bytes() objects and treat them as COPY data. This optimization allows the avoidance of creating CopyData messages for each COPY line. Arguably, at the client3 level, this optimization is inappropriate. However, it is unlikely for there to be another case where this implicit message typing is desirable for PQv3. It is likely that any client3 extensions would take on formal element3 subclasses, and therefore not require the exercise of such functionality. Additionally, if implicit typing were to be desireable, it would require additional configuration of some sort even in the absence of this optimization. Perhaps the parameterization of the implied message type would be appropriate compensation for advanced applications..
1 parent d12d446 commit ea780ac

10 files changed

Lines changed: 674 additions & 130 deletions

File tree

‎postgresql/driver/pq3.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1067,7 +1067,7 @@ def _load_copy_chunks(self, chunks):
10671067
raise e
10681068

10691069
for chunk in chunks:
1070-
x.messages = [element.CopyData(l) for l in chunk]
1070+
x.messages = list(chunk)
10711071
while x.messages is not x.CopyFailSequence:
10721072
self.database._pq_step()
10731073
x.messages = x.CopyDoneSequence

‎postgresql/protocol/client3.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,20 @@
1010
from . import element3 as element
1111
from . import xact3 as xact
1212
from .buffer import pq_message_stream
13+
from .typstruct import long_pack
14+
15+
def cat_messages(messages):
16+
blen = bytes.__len__
17+
lpack = long_pack
18+
return b''.join([
19+
x.bytes() if x.__class__ is not bytes else (
20+
b'd' + lpack(blen(x) + 4) + x
21+
) for x in messages
22+
])
23+
try:
24+
from .optimized import cat_messages
25+
except ImportError:
26+
pass
1327

1428
class Connection(object):
1529
"""
@@ -290,12 +304,12 @@ def send_message_data(self):
290304
def standard_write_messages(self, messages):
291305
'protocol message writer'
292306
if self.writing is not self.written:
293-
self.message_data += b''.join([x.bytes() for x in self.writing])
307+
self.message_data += cat_messages(self.writing)
294308
self.written = self.writing
295309

296310
if messages is not self.writing:
297311
self.writing = messages
298-
self.message_data += b''.join([x.bytes() for x in self.writing])
312+
self.message_data += cat_messages(self.writing)
299313
self.written = self.writing
300314
return self.send_message_data()
301315
write_messages = standard_write_messages

‎postgresql/protocol/element3.py‎

Lines changed: 51 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,26 @@
66
import sys
77
import os
88
import pprint
9-
from struct import pack, unpack, Struct
9+
from struct import unpack, Struct
10+
from .message_types import message_types
11+
from .typstruct import ushort_pack, ushort_unpack, ulong_pack, ulong_unpack
12+
13+
def pack_tuple_data(atts):
14+
return b''.join([
15+
b'\xff\xff\xff\xff'
16+
if x is None
17+
else (ulong_pack(len(x)) + x)
18+
for x in atts
19+
])
1020

1121
try:
12-
from .optimized import parse_tuple_message
22+
from .optimized import parse_tuple_message, pack_tuple_data
1323
except ImportError:
1424
pass
1525

16-
from .message_types import message_types
17-
1826
StringFormat = b'\x00\x00'
1927
BinaryFormat = b'\x00\x01'
2028

21-
byte = Struct("!B")
22-
ushort = Struct("!H")
23-
ulong = Struct("!L")
24-
2529
class Message(object):
2630
bytes_struct = Struct("!cL")
2731
__slots__ = ()
@@ -125,11 +129,11 @@ def serialize(self):
125129

126130
@classmethod
127131
def parse(typ, data):
128-
if ulong.unpack(data[1:5])[0] != len(data) - 1:
132+
if ulong_unpack(data[1:5]) != len(data) - 1:
129133
raise ValueError(
130134
"invalid wire message where data is %d bytes and " \
131135
"internal size stamp is %d bytes" %(
132-
len(data), ulong.unpack(data[1:5])[0] + 1
136+
len(data), ulong_unpack(data[1:5]) + 1
133137
)
134138
)
135139
return typ((data[0:1], data[5:]))
@@ -162,13 +166,13 @@ def __init__(self, pid, relation, parameter = b''):
162166
self.parameter = parameter
163167

164168
def serialize(self):
165-
return ulong.pack(self.pid) + \
169+
return ulong_pack(self.pid) + \
166170
self.relation + b'\x00' + \
167171
self.parameter + b'\x00'
168172

169173
@classmethod
170174
def parse(typ, data):
171-
pid = ulong.unpack(data[0:4])[0]
175+
pid = ulong_unpack(data[0:4])
172176
relname, param, nothing = data[4:].split(b'\x00', 2)
173177
return typ(pid, relname, param)
174178

@@ -363,13 +367,13 @@ def __init__(self, datum):
363367

364368
def serialize(self):
365369
return self.result is None and b'\xff\xff\xff\xff' or \
366-
ulong.pack(len(self.result)) + self.result
370+
ulong_pack(len(self.result)) + self.result
367371

368372
@classmethod
369373
def parse(typ, data):
370374
if data == b'\xff\xff\xff\xff':
371375
return typ(None)
372-
size = ulong.unpack(data[0:4])[0]
376+
size = ulong_unpack(data[0:4])
373377
data = data[4:]
374378
if size != len(data):
375379
raise ValueError(
@@ -385,11 +389,11 @@ class AttributeTypes(TupleMessage):
385389
__slots__ = ()
386390

387391
def serialize(self):
388-
return ushort.pack(len(self)) + b''.join([ulong.pack(x) for x in self])
392+
return ushort_pack(len(self)) + b''.join([ulong_pack(x) for x in self])
389393

390394
@classmethod
391395
def parse(typ, data):
392-
ac = ushort.unpack(data[0:2])[0]
396+
ac = ushort_unpack(data[0:2])
393397
args = data[2:]
394398
if len(args) != ac * 4:
395399
raise ValueError("invalid argument type data size")
@@ -405,14 +409,14 @@ def keys(self):
405409
return [x[0] for x in self]
406410

407411
def serialize(self):
408-
return ushort.pack(len(self)) + b''.join([
412+
return ushort_pack(len(self)) + b''.join([
409413
x[0] + b'\x00' + self.struct.pack(*x[1:])
410414
for x in self
411415
])
412416

413417
@classmethod
414418
def parse(typ, data):
415-
ac = ushort.unpack(data[0:2])[0]
419+
ac = ushort_unpack(data[0:2])
416420
atts = []
417421
data = data[2:]
418422
ca = 0
@@ -433,14 +437,11 @@ class Tuple(TupleMessage):
433437
__slots__ = ()
434438

435439
def serialize(self):
436-
return ushort.pack(len(self)) + b''.join([
437-
x is None and b'\xff\xff\xff\xff' or ulong.pack(len(x)) + bytes(x)
438-
for x in self
439-
])
440+
return ushort_pack(len(self)) + pack_tuple_data(self)
440441

441442
@classmethod
442443
def parse(typ, data):
443-
natts = ushort.unpack(data[0:2])[0]
444+
natts = ushort_unpack(data[0:2])
444445
atts = list()
445446
offset = 2
446447

@@ -451,7 +452,7 @@ def parse(typ, data):
451452
if size == b'\xff\xff\xff\xff':
452453
att = None
453454
else:
454-
al = ulong.unpack(size)[0]
455+
al = ulong_unpack(size)
455456
ao = offset
456457
offset = ao + al
457458
att = data[ao:offset]
@@ -494,7 +495,7 @@ def serialize(self):
494495

495496
def bytes(self):
496497
data = self.serialize()
497-
return ulong.pack(len(data) + 4) + self.serialize()
498+
return ulong_pack(len(data) + 4) + self.serialize()
498499

499500
@classmethod
500501
def parse(typ, data):
@@ -514,7 +515,7 @@ def __new__(typ):
514515

515516
def bytes(self):
516517
data = self.serialize()
517-
return ulong.pack(len(data) + 4) + data
518+
return ulong_pack(len(data) + 4) + data
518519

519520
def serialize(self):
520521
return self.packed_version
@@ -545,7 +546,7 @@ def serialize(self):
545546

546547
def bytes(self):
547548
data = self.serialize()
548-
return ulong.pack(len(data) + 4) + data
549+
return ulong_pack(len(data) + 4) + data
549550

550551
@classmethod
551552
def parse(typ, data):
@@ -598,11 +599,11 @@ def __init__(self, request, salt):
598599
self.salt = salt
599600

600601
def serialize(self):
601-
return ulong.pack(self.request) + self.salt
602+
return ulong_pack(self.request) + self.salt
602603

603604
@classmethod
604605
def parse(typ, data):
605-
return typ(ulong.unpack(data[0:4])[0], data[4:])
606+
return typ(ulong_unpack(data[0:4]), data[4:])
606607

607608
class Password(StringMessage):
608609
'Password supplement'
@@ -648,17 +649,17 @@ def __init__(self, name, statement, argtypes):
648649
@classmethod
649650
def parse(typ, data):
650651
name, statement, args = data.split(b'\x00', 2)
651-
ac = ushort.unpack(args[0:2])[0]
652+
ac = ushort_unpack(args[0:2])
652653
args = args[2:]
653654
if len(args) != ac * 4:
654655
raise ValueError("invalid argument type data")
655656
at = unpack('!%dL'%(ac,), args)
656657
return typ(name, statement, at)
657658

658659
def serialize(self):
659-
ac = ushort.pack(len(self.argtypes))
660+
ac = ushort_pack(len(self.argtypes))
660661
return self.name + b'\x00' + self.statement + b'\x00' + ac + b''.join([
661-
ulong.pack(x) for x in self.argtypes
662+
ulong_pack(x) for x in self.argtypes
662663
])
663664

664665
class Bind(Message):
@@ -685,14 +686,9 @@ def __init__(self, name, statement, aformats, arguments, rformats):
685686

686687
def serialize(self):
687688
args = self.arguments
688-
ac = ushort.pack(len(args))
689-
ad = b''.join([
690-
b'\xff\xff\xff\xff'
691-
if x is None
692-
else (ulong.pack(len(x)) + x)
693-
for x in args
694-
])
695-
rfc = ushort.pack(len(self.rformats))
689+
ac = ushort_pack(len(args))
690+
ad = pack_tuple_data(tuple(args))
691+
rfc = ushort_pack(len(self.rformats))
696692
return \
697693
self.name + b'\x00' + self.statement + b'\x00' + \
698694
ac + b''.join(self.aformats) + ac + ad + rfc + \
@@ -701,11 +697,11 @@ def serialize(self):
701697
@classmethod
702698
def parse(typ, message_data):
703699
name, statement, data = message_data.split(b'\x00', 2)
704-
ac = ushort.unpack(data[:2])[0]
700+
ac = ushort_unpack(data[:2])
705701
offset = 2 + (2 * ac)
706702
aformats = unpack(("2s" * ac), data[2:offset])
707703

708-
natts = ushort.unpack(data[offset:offset+2])[0]
704+
natts = ushort_unpack(data[offset:offset+2])
709705
args = list()
710706
offset += 2
711707

@@ -716,14 +712,14 @@ def parse(typ, message_data):
716712
if size == b'\xff\xff\xff\xff':
717713
att = None
718714
else:
719-
al = ulong.unpack(size)[0]
715+
al = ulong_unpack(size)
720716
ao = offset
721717
offset = ao + al
722718
att = data[ao:offset]
723719
args.append(att)
724720
natts -= 1
725721

726-
rfc = ushort.unpack(data[offset:offset+2])[0]
722+
rfc = ushort_unpack(data[offset:offset+2])
727723
ao = offset + 2
728724
offset = ao + (2 * rfc)
729725
rformats = unpack(("2s" * rfc), data[ao:offset])
@@ -740,12 +736,12 @@ def __init__(self, name, max = 0):
740736
self.max = max
741737

742738
def serialize(self):
743-
return self.name + pack("!BL", 0, self.max)
739+
return self.name + b'\x00' + ulong_pack(self.max)
744740

745741
@classmethod
746742
def parse(typ, data):
747743
name, max = data.split(b'\x00', 1)
748-
return typ(name, ulong.unpack(max)[0])
744+
return typ(name, ulong_unpack(max))
749745

750746
class Describe(StringMessage):
751747
"""Describe a Portal or Prepared Statement"""
@@ -813,23 +809,20 @@ def __init__(self, oid, aformats, args, rformat = StringFormat):
813809
self.rformat = rformat
814810

815811
def serialize(self):
816-
ac = ushort.pack(len(self.arguments))
817-
return ulong.pack(self.oid) + \
812+
ac = ushort_pack(len(self.arguments))
813+
return ulong_pack(self.oid) + \
818814
ac + b''.join(self.aformats) + \
819-
ac + b''.join([
820-
(x is None) and b'\xff\xff\xff\xff' or ulong.pack(len(x)) + x
821-
for x in self.arguments
822-
]) + self.rformat
815+
ac + pack_tuple_data(tuple(self.arguments)) + self.rformat
823816

824817
@classmethod
825818
def parse(typ, data):
826-
oid = ulong.unpack(data[0:4])[0]
819+
oid = ulong_unpack(data[0:4])
827820

828-
ac = ushort.unpack(data[4:6])[0]
821+
ac = ushort_unpack(data[4:6])
829822
offset = 6 + (2 * ac)
830823
aformats = unpack(("2s" * ac), data[6:offset])
831824

832-
natts = ushort.unpack(data[offset:offset+2])[0]
825+
natts = ushort_unpack(data[offset:offset+2])
833826
args = list()
834827
offset += 2
835828

@@ -840,7 +833,7 @@ def parse(typ, data):
840833
if size == b'\xff\xff\xff\xff':
841834
att = None
842835
else:
843-
al = ulong.unpack(size)[0]
836+
al = ulong_unpack(size)
844837
ao = offset
845838
offset = ao + al
846839
att = data[ao:offset]
@@ -860,7 +853,7 @@ def __init__(self, format, formats):
860853

861854
def serialize(self):
862855
return self.struct.pack(self.format, len(self.formats)) + b''.join([
863-
ushort.pack(x) for x in self.formats
856+
ushort_pack(x) for x in self.formats
864857
])
865858

866859
@classmethod
@@ -870,7 +863,7 @@ def parse(typ, data):
870863
if len(formats_str) != natts * 2:
871864
raise ValueError("number of formats and data do not match up")
872865
return typ(format, [
873-
ushort.unpack(formats_str[x:x+2])[0] for x in range(0, natts * 2, 2)
866+
ushort_unpack(formats_str[x:x+2]) for x in range(0, natts * 2, 2)
874867
])
875868

876869
class CopyToBegin(CopyBegin):

0 commit comments

Comments
 (0)