Skip to content

Commit b138be5

Browse files
author
James William Pye
committed
Optimize int2 & int4 packing, and remove dependency on ntohl(winsock).
1 parent 65f634c commit b138be5

6 files changed

Lines changed: 275 additions & 20 deletions

File tree

‎postgresql/protocol/optimized/buffer.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ p_length(PyObject *self)
220220
break;
221221
p_seek(&p, copy_amount);
222222

223-
msg_length = ntohl(*((uint32_t *) (header + 1)));
223+
msg_length = local_ntohl(*((uint32_t *) (header + 1)));
224224
if (msg_length < 4)
225225
{
226226
PyErr_Format(PyExc_ValueError,
@@ -262,7 +262,7 @@ p_build_tuple(struct p_place *p)
262262
return(NULL);
263263
p_seek(p, copy_amount);
264264

265-
msg_length = ntohl(*((uint32_t *) (header + 1)));
265+
msg_length = local_ntohl(*((uint32_t *) (header + 1)));
266266
if (msg_length < 4)
267267
{
268268
PyErr_Format(PyExc_ValueError,
@@ -462,7 +462,7 @@ p_has_message(PyObject *self)
462462
}
463463
p_seek(&p, copy_amount);
464464

465-
msg_length = ntohl(*((uint32_t *) (header + 1)));
465+
msg_length = local_ntohl(*((uint32_t *) (header + 1)));
466466
if (msg_length < 4)
467467
{
468468
PyErr_Format(PyExc_ValueError,

‎postgresql/protocol/optimized/module.c‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@
44
*
55
*//*
66
* Optimizations for protocol modules.
7+
*
8+
* This module.c file ties together other classified C source.
9+
* Each C-file describing the part of the protocol package that it
10+
* covers. It merely uses CPP includes to bring them into this
11+
* file and then uses some CPP macros to expand the definitions
12+
* in each file.
713
*/
814
#include <stdint.h>
9-
#ifdef WIN32
10-
#include <winsock.h>
11-
#else
12-
#include <sys/types.h>
13-
#include <netinet/in.h>
14-
#endif
1515
#include <Python.h>
1616
#include <structmember.h>
1717

@@ -20,6 +20,7 @@
2020
* Initialized in PyInit_optimized.
2121
*/
2222
static PyObject *message_types = NULL;
23+
static long (*local_ntohl)(long) = NULL;
2324

2425

2526
#include "typio.c"
@@ -51,12 +52,13 @@ PyInit_optimized(void)
5152
PyObject *mod;
5253
PyObject *msgtypes;
5354
PyObject *fromlist, *fromstr;
55+
long l = 1;
5456

5557
mod = PyModule_Create(&optimized_module);
5658
if (mod == NULL)
5759
return(NULL);
5860

59-
/* cpp abuse */
61+
/* cpp abuse; ready types */
6062
#define mTYPE(name) \
6163
if (PyType_Ready(&name##_Type) < 0) \
6264
goto cleanup; \
@@ -68,6 +70,16 @@ PyInit_optimized(void)
6870
include_buffer_types
6971
#undef mTYPE
7072

73+
if (((char *) &l)[0] == 1)
74+
{
75+
/* little */
76+
local_ntohl = swap_long;
77+
}
78+
else
79+
{
80+
/* big */
81+
local_ntohl = return_long;
82+
}
7183

7284
/*
7385
* Get the message_types tuple to type "instantiation".

‎postgresql/protocol/optimized/typio.c‎

Lines changed: 197 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,208 @@
11
/*
22
* copyright 2009, James William Pye
33
* http://python.projects.postgresql.org
4-
*
54
*/
65
#define include_typio_functions \
76
mFUNC(process_tuple, METH_VARARGS, \
87
"process the items in the second argument " \
98
"with the corresponding items in the first argument.") \
9+
mFUNC(int2_pack, METH_O, "PyInt to serialized, int2") \
10+
mFUNC(int2_unpack, METH_O, "PyInt from serialized, int2") \
11+
mFUNC(int4_pack, METH_O, "PyInt to serialized, int4") \
12+
mFUNC(int4_unpack, METH_O, "PyInt from serialized, int4") \
13+
mFUNC(swap_int2_pack, METH_O, "PyInt to swapped serialized, int2") \
14+
mFUNC(swap_int2_unpack, METH_O, "PyInt from swapped serialized, int2") \
15+
mFUNC(swap_int4_pack, METH_O, "PyInt to swapped serialized, int4") \
16+
mFUNC(swap_int4_unpack, METH_O, "PyInt from swapped serialized, int4") \
17+
18+
#define SHORT_MAX ((2<<14)-1)
19+
#define SHORT_MIN (-(2<<14))
20+
21+
/*
22+
* Define the swap functionality.
23+
*/
24+
#define swap2(CP) do{register char c; \
25+
c=CP[1];CP[1]=CP[0];CP[0]=c;\
26+
}while(0)
27+
#define swap4(P) do{register char c; \
28+
c=P[3];P[3]=P[0];P[0]=c;\
29+
c=P[2];P[2]=P[1];P[1]=c;\
30+
}while(0)
31+
#define swap8(P) do{register char c; \
32+
c=P[7];P[7]=P[0];P[0]=c;\
33+
c=P[6];P[6]=P[1];P[1]=c;\
34+
c=P[5];P[5]=P[2];P[2]=c;\
35+
c=P[4];P[4]=P[3];P[3]=c;\
36+
}while(0)
37+
38+
static long
39+
swap_long(long l)
40+
{
41+
swap4(((char *) &l));
42+
return(l);
43+
}
44+
45+
static long
46+
return_long(long l)
47+
{
48+
return(l);
49+
}
50+
51+
static PyObject *
52+
int2_pack(PyObject *self, PyObject *arg)
53+
{
54+
long l;
55+
short s;
56+
57+
l = PyLong_AsLong(arg);
58+
if (PyErr_Occurred())
59+
return(NULL);
60+
61+
if (l > SHORT_MAX || l < SHORT_MIN)
62+
{
63+
PyErr_Format(PyExc_OverflowError,
64+
"long '%d' overflows int2", l
65+
);
66+
return(NULL);
67+
}
68+
69+
s = (short) l;
70+
return(PyBytes_FromStringAndSize((const char *) &s, 2));
71+
}
72+
static PyObject *
73+
swap_int2_pack(PyObject *self, PyObject *arg)
74+
{
75+
long l;
76+
short s;
77+
78+
l = PyLong_AsLong(arg);
79+
if (PyErr_Occurred())
80+
return(NULL);
81+
if (l > SHORT_MAX || l < SHORT_MIN)
82+
{
83+
PyErr_SetString(PyExc_OverflowError, "long too big or small for int2");
84+
return(NULL);
85+
}
86+
87+
s = (short) l;
88+
swap2(((char *) &s));
89+
return(PyBytes_FromStringAndSize((const char *) &s, 2));
90+
}
91+
92+
static PyObject *
93+
int2_unpack(PyObject *self, PyObject *arg)
94+
{
95+
char *c;
96+
short *i;
97+
long l;
98+
Py_ssize_t len;
99+
PyObject *rob;
100+
101+
c = PyBytes_AsString(arg);
102+
if (PyErr_Occurred())
103+
return(NULL);
104+
105+
len = PyBytes_Size(arg);
106+
if (len != 2)
107+
{
108+
PyErr_SetString(PyExc_ValueError, "invalid size of data for int2_unpack");
109+
return(NULL);
110+
}
111+
112+
i = (short *) c;
113+
l = (long) *i;
114+
rob = PyLong_FromLong(l);
115+
return(rob);
116+
}
117+
static PyObject *
118+
swap_int2_unpack(PyObject *self, PyObject *arg)
119+
{
120+
char *c;
121+
short s;
122+
long l;
123+
Py_ssize_t len;
124+
PyObject *rob;
125+
126+
c = PyBytes_AsString(arg);
127+
if (PyErr_Occurred())
128+
return(NULL);
129+
130+
len = PyBytes_Size(arg);
131+
if (len != 2)
132+
{
133+
PyErr_SetString(PyExc_ValueError, "invalid size of data for int2_unpack");
134+
return(NULL);
135+
}
136+
137+
s = *((short *) c);
138+
swap2(((char *) &s));
139+
l = (long) s;
140+
rob = PyLong_FromLong(l);
141+
return(rob);
142+
}
143+
144+
static PyObject *
145+
int4_pack(PyObject *self, PyObject *arg)
146+
{
147+
long l;
148+
l = PyLong_AsLong(arg);
149+
if (PyErr_Occurred())
150+
return(NULL);
151+
return(PyBytes_FromStringAndSize((const char *) &l, 4));
152+
}
153+
static PyObject *
154+
swap_int4_pack(PyObject *self, PyObject *arg)
155+
{
156+
long l;
157+
l = PyLong_AsLong(arg);
158+
if (PyErr_Occurred())
159+
return(NULL);
160+
swap4(((char *) &l));
161+
return(PyBytes_FromStringAndSize((const char *) &l, 4));
162+
}
163+
164+
static PyObject *
165+
int4_unpack(PyObject *self, PyObject *arg)
166+
{
167+
char *c;
168+
long l;
169+
Py_ssize_t len;
170+
171+
len = PyBytes_Size(arg);
172+
if (len != 4)
173+
{
174+
PyErr_SetString(PyExc_ValueError, "invalid size of data for int4_unpack");
175+
return(NULL);
176+
}
177+
c = PyBytes_AsString(arg);
178+
if (PyErr_Occurred())
179+
return(NULL);
180+
l = *((long *) c);
181+
182+
return(PyLong_FromLong(l));
183+
}
184+
static PyObject *
185+
swap_int4_unpack(PyObject *self, PyObject *arg)
186+
{
187+
char *c;
188+
long l;
189+
Py_ssize_t len;
190+
191+
c = PyBytes_AsString(arg);
192+
if (PyErr_Occurred())
193+
return(NULL);
194+
195+
len = PyBytes_Size(arg);
196+
if (len != 4)
197+
{
198+
PyErr_SetString(PyExc_ValueError, "invalid size of data for int4_unpack");
199+
return(NULL);
200+
}
201+
202+
l = *((long *) c);
203+
swap4(((char *) &l));
204+
return(PyLong_FromLong(l));
205+
}
10206

11207
/*
12208
* process the tuple with the associated callables while
@@ -165,4 +361,3 @@ process_tuple(PyObject *self, PyObject *args)
165361

166362
return(rob);
167363
}
168-

‎postgresql/protocol/typstruct.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
null_sequence = b'\xff\xff\xff\xff'
2626

27+
# Always to and from network order.
2728
def mk_pack(x):
2829
'Create a pair, (pack, unpack) for the given `struct` format.'
2930
s = struct.Struct('!' + x)
@@ -68,7 +69,7 @@ def mktime64(seconds_ms):
6869
longlong_pack, longlong_unpack = mk_pack("q")
6970
long_pack, long_unpack = mk_pack("l")
7071
ulong_pack, ulong_unpack = mk_pack("L")
71-
byte_pack, byte_unpack = mk_pack("B")
72+
byte_pack, byte_unpack = lambda x: bytes((x,)), lambda x: x[0]
7273
short_pack, short_unpack = mk_pack("h")
7374
ushort_pack, ushort_unpack = mk_pack("H")
7475
double_pack, double_unpack = mk_pack("d")
@@ -87,6 +88,18 @@ def mktime64(seconds_ms):
8788

8889
hhhh_pack, hhhh_unpack = mk_pack("hhhh")
8990

91+
try:
92+
from sys import byteorder as bo
93+
if bo == 'little':
94+
from .optimized import swap_int2_unpack as short_unpack, swap_int2_pack as short_pack
95+
from .optimized import swap_int4_unpack as long_unpack, swap_int4_pack as long_pack
96+
else:
97+
from .optimized import int2_unpack as short_unpack, int2_pack as short_pack
98+
from .optimized import int4_unpack as long_unpack, int4_pack as long_pack
99+
del bo
100+
except ImportError:
101+
pass
102+
90103
int2_pack, int2_unpack = short_pack, short_unpack
91104
int4_pack, int4_unpack = long_pack, long_unpack
92105
int8_pack, int8_unpack = longlong_pack, longlong_unpack

‎postgresql/release/distutils.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,9 @@
9999
extensions_data = {
100100
'protocol.optimized' : {
101101
'sources' : [os.path.join('protocol', 'optimized', 'module.c')],
102-
'libraries' : (sys.platform == 'win32' and ['ws2_32'] or []),
103102
},
104103
'python.optimized' : {
105104
'sources' : [os.path.join('python', 'optimized.c')],
106-
'libraries' : (sys.platform == 'win32' and ['ws2_32'] or []),
107105
},
108106
}
109107

@@ -134,7 +132,7 @@ def prefixed_extensions(
134132
yield Extension(
135133
pkg_prefix + mod,
136134
[os.path.join(path_prefix, src) for src in data['sources']],
137-
libraries = data['libraries']
135+
libraries = data.get('libraries', ())
138136
)
139137

140138
def prefixed_packages(

0 commit comments

Comments
 (0)