-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathpython37_internals.h
More file actions
471 lines (410 loc) · 17.1 KB
/
Copy pathpython37_internals.h
File metadata and controls
471 lines (410 loc) · 17.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
/* -*- mode: c++; indent-tabs-mode: nil -*- */
/*
python37_internals.h
Qore Programming Language
Copyright 2020 - 2022 Qore Technologies, s.r.o.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef _QORE_PYTHON_INTERNALS_H
#define _QORE_PYTHON_INTERNALS_H
#include <pystate.h>
#include <dynamic_annotations.h>
inline int PyThreadState_GetRecursionLimit(PyThreadState* state) {
return state->recursion_depth;
}
inline void PyThreadState_UpdateRecursionLimit(PyThreadState* state, int new_limit) {
state->recursion_depth = new_limit;
}
typedef struct _Py_atomic_address {
uintptr_t _value;
} _Py_atomic_address;
typedef struct _Py_atomic_int {
int _value;
} _Py_atomic_int;
struct _pending_calls {
unsigned long main_thread;
PyThread_type_lock lock;
/* Request for running pending calls. */
_Py_atomic_int calls_to_do;
/* Request for looking at the `async_exc` field of the current
thread state.
Guarded by the GIL. */
int async_exc;
#define NPENDINGCALLS 32
struct {
int (*func)(void *);
void *arg;
} calls[NPENDINGCALLS];
int first;
int last;
};
#define FORCE_SWITCHING
#define PyMUTEX_T pthread_mutex_t
#define PyCOND_T pthread_cond_t
struct _gil_runtime_state {
/* microseconds (the Python API uses seconds, though) */
unsigned long interval;
/* Last PyThreadState holding / having held the GIL. This helps us
know whether anyone else was scheduled after we dropped the GIL. */
_Py_atomic_address last_holder;
/* Whether the GIL is already taken (-1 if uninitialized). This is
atomic because it can be read without any lock taken in ceval.c. */
_Py_atomic_int locked;
/* Number of GIL switches since the beginning. */
unsigned long switch_number;
/* This condition variable allows one or several threads to wait
until the GIL is released. In addition, the mutex also protects
the above variables. */
PyCOND_T cond;
PyMUTEX_T mutex;
#ifdef FORCE_SWITCHING
/* This condition variable helps the GIL-releasing thread wait for
a GIL-awaiting thread to be scheduled and take the GIL. */
PyCOND_T switch_cond;
PyMUTEX_T switch_mutex;
#endif
};
struct _ceval_runtime_state {
int recursion_limit;
/* Records whether tracing is on for any thread. Counts the number
of threads for which tstate->c_tracefunc is non-NULL, so if the
value is 0, we know we don't have to check this thread's
c_tracefunc. This speeds up the if statement in
PyEval_EvalFrameEx() after fast_next_opcode. */
int tracing_possible;
/* This single variable consolidates all requests to break out of
the fast path in the eval loop. */
_Py_atomic_int eval_breaker;
/* Request for dropping the GIL */
_Py_atomic_int gil_drop_request;
struct _pending_calls pending;
struct _gil_runtime_state gil;
};
struct _warnings_runtime_state {
/* Both 'filters' and 'onceregistry' can be set in warnings.py;
get_warnings_attr() will reset these variables accordingly. */
PyObject *filters; /* List */
PyObject *once_registry; /* Dict */
PyObject *default_action; /* String */
long filters_version;
};
/* If we change this, we need to change the default value in the
signature of gc.collect. */
#define NUM_GENERATIONS 3
struct gc_generation {
PyGC_Head head;
int threshold; /* collection threshold */
int count; /* count of allocations or collections of younger
generations */
};
/* Running stats per generation */
struct gc_generation_stats {
/* total number of collections */
Py_ssize_t collections;
/* total number of collected objects */
Py_ssize_t collected;
/* total number of uncollectable objects (put into gc.garbage) */
Py_ssize_t uncollectable;
};
struct _gc_runtime_state {
/* List of objects that still need to be cleaned up, singly linked
* via their gc headers' gc_prev pointers. */
PyObject *trash_delete_later;
/* Current call-stack depth of tp_dealloc calls. */
int trash_delete_nesting;
int enabled;
int debug;
/* linked lists of container objects */
struct gc_generation generations[NUM_GENERATIONS];
PyGC_Head *generation0;
/* a permanent generation which won't be collected */
struct gc_generation permanent_generation;
struct gc_generation_stats generation_stats[NUM_GENERATIONS];
/* true if we are currently running the collector */
int collecting;
/* list of uncollectable objects */
PyObject *garbage;
/* a list of callbacks to be invoked when collection is performed */
PyObject *callbacks;
/* This is the number of objects that survived the last full
collection. It approximates the number of long lived objects
tracked by the GC.
(by "full collection", we mean a collection of the oldest
generation). */
Py_ssize_t long_lived_total;
/* This is the number of objects that survived all "non-full"
collections, and are awaiting to undergo a full collection for
the first time. */
Py_ssize_t long_lived_pending;
};
struct _gilstate_runtime_state {
int check_enabled;
/* Assuming the current thread holds the GIL, this is the
PyThreadState for the current thread. */
_Py_atomic_address tstate_current;
PyThreadFrameGetter getframe;
/* The single PyInterpreterState used by this process'
GILState implementation
*/
/* TODO: Given interp_main, it may be possible to kill this ref */
PyInterpreterState *autoInterpreterState;
Py_tss_t autoTSSkey;
};
typedef struct pyruntimestate {
int initialized;
int core_initialized;
PyThreadState *finalizing;
struct pyinterpreters {
PyThread_type_lock mutex;
PyInterpreterState *head;
PyInterpreterState *main;
/* _next_interp_id is an auto-numbered sequence of small
integers. It gets initialized in _PyInterpreterState_Init(),
which is called in Py_Initialize(), and used in
PyInterpreterState_New(). A negative interpreter ID
indicates an error occurred. The main interpreter will
always have an ID of 0. Overflow results in a RuntimeError.
If that becomes a problem later then we can adjust, e.g. by
using a Python int. */
int64_t next_id;
} interpreters;
#define NEXITFUNCS 32
void (*exitfuncs[NEXITFUNCS])(void);
int nexitfuncs;
struct _gc_runtime_state gc;
struct _warnings_runtime_state warnings;
struct _ceval_runtime_state ceval;
struct _gilstate_runtime_state gilstate;
// XXX Consolidate globals found via the check-c-globals script.
} _PyRuntimeState;
DLLLOCAL extern _PyRuntimeState _PyRuntime;
#if defined(__GNUC__) && (defined(__i386__) || defined(__amd64))
typedef enum _Py_memory_order {
_Py_memory_order_relaxed,
_Py_memory_order_acquire,
_Py_memory_order_release,
_Py_memory_order_acq_rel,
_Py_memory_order_seq_cst
} _Py_memory_order;
static __inline__ void
_Py_atomic_signal_fence(_Py_memory_order order)
{
if (order != _Py_memory_order_relaxed)
__asm__ volatile("":::"memory");
}
static __inline__ void
_Py_atomic_thread_fence(_Py_memory_order order)
{
if (order != _Py_memory_order_relaxed)
__asm__ volatile("mfence":::"memory");
}
/* Tell the race checker about this operation's effects. */
static __inline__ void
_Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order)
{
(void)address; /* shut up -Wunused-parameter */
switch(order) {
case _Py_memory_order_release:
case _Py_memory_order_acq_rel:
case _Py_memory_order_seq_cst:
_Py_ANNOTATE_HAPPENS_BEFORE(address);
break;
case _Py_memory_order_relaxed:
case _Py_memory_order_acquire:
break;
}
switch(order) {
case _Py_memory_order_acquire:
case _Py_memory_order_acq_rel:
case _Py_memory_order_seq_cst:
_Py_ANNOTATE_HAPPENS_AFTER(address);
break;
case _Py_memory_order_relaxed:
case _Py_memory_order_release:
break;
}
}
#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
__extension__ ({ \
__typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
__typeof__(atomic_val->_value) result; \
volatile __typeof__(result) *volatile_data = &atomic_val->_value; \
_Py_memory_order order = ORDER; \
_Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
\
/* Perform the operation. */ \
_Py_ANNOTATE_IGNORE_READS_BEGIN(); \
switch(order) { \
case _Py_memory_order_release: \
case _Py_memory_order_acq_rel: \
case _Py_memory_order_seq_cst: \
/* Loads on x86 are not releases by default, so need a */ \
/* thread fence. */ \
_Py_atomic_thread_fence(_Py_memory_order_release); \
break; \
default: \
/* No fence */ \
break; \
} \
result = *volatile_data; \
switch(order) { \
case _Py_memory_order_acquire: \
case _Py_memory_order_acq_rel: \
case _Py_memory_order_seq_cst: \
/* Loads on x86 are automatically acquire operations so */ \
/* can get by with just a compiler fence. */ \
_Py_atomic_signal_fence(_Py_memory_order_acquire); \
break; \
default: \
/* No fence */ \
break; \
} \
_Py_ANNOTATE_IGNORE_READS_END(); \
result; \
})
#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
__extension__ ({ \
__typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
__typeof__(atomic_val->_value) new_val = NEW_VAL;\
volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \
_Py_memory_order order = ORDER; \
_Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
\
/* Perform the operation. */ \
_Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \
switch(order) { \
case _Py_memory_order_release: \
_Py_atomic_signal_fence(_Py_memory_order_release); \
/* fallthrough */ \
case _Py_memory_order_relaxed: \
*volatile_data = new_val; \
break; \
\
case _Py_memory_order_acquire: \
case _Py_memory_order_acq_rel: \
case _Py_memory_order_seq_cst: \
__asm__ volatile("xchg %0, %1" \
: "+r"(new_val) \
: "m"(atomic_val->_value) \
: "memory"); \
break; \
} \
_Py_ANNOTATE_IGNORE_WRITES_END(); \
})
#elif defined(__GNUC__) && (defined(__arm__))
# include <atomic>
typedef enum _Py_memory_order {
_Py_memory_order_relaxed = std::memory_order_relaxed,
_Py_memory_order_acquire = std::memory_order_acquire,
_Py_memory_order_release = std::memory_order_release,
_Py_memory_order_acq_rel = std::memory_order_acq_rel,
_Py_memory_order_seq_cst = std::memory_order_seq_cst
} _Py_memory_order;
#define atomic_load_explicit(PTR, MO) \
__extension__ \
({ \
__auto_type __atomic_load_ptr = (PTR); \
__typeof__ (*__atomic_load_ptr) __atomic_load_tmp; \
__atomic_load (__atomic_load_ptr, &__atomic_load_tmp, (MO)); \
__atomic_load_tmp; \
})
#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER)
#define atomic_store_explicit(PTR, VAL, MO) \
__extension__ \
({ \
__auto_type __atomic_store_ptr = (PTR); \
__typeof__ (*__atomic_store_ptr) __atomic_store_tmp = (VAL); \
__atomic_store (__atomic_store_ptr, &__atomic_store_tmp, (MO)); \
})
#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)
#elif defined(__GNUC__)
// assume all other architectures use standard atomic functions
# include <atomic>
typedef enum _Py_memory_order {
_Py_memory_order_relaxed = std::memory_order_relaxed,
_Py_memory_order_acquire = std::memory_order_acquire,
_Py_memory_order_release = std::memory_order_release,
_Py_memory_order_acq_rel = std::memory_order_acq_rel,
_Py_memory_order_seq_cst = std::memory_order_seq_cst
} _Py_memory_order;
#define atomic_load_explicit(PTR, MO) \
__extension__ \
({ \
auto __atomic_load_ptr = (PTR); \
__typeof__ (*__atomic_load_ptr) __atomic_load_tmp; \
__atomic_load (__atomic_load_ptr, &__atomic_load_tmp, (MO)); \
__atomic_load_tmp; \
})
#define atomic_store_explicit(PTR, VAL, MO) \
__extension__ \
({ \
auto __atomic_store_ptr = (PTR); \
__typeof__ (*__atomic_store_ptr) __atomic_store_tmp = (VAL); \
__atomic_store (__atomic_store_ptr, &__atomic_store_tmp, (MO)); \
})
#define atomic_store_explicit(PTR, VAL, MO) \
__extension__ \
({ \
auto __atomic_store_ptr = (PTR); \
__typeof__ (*__atomic_store_ptr) __atomic_store_tmp = (VAL); \
__atomic_store (__atomic_store_ptr, &__atomic_store_tmp, (MO)); \
})
#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER)
#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)
#endif
#define _Py_atomic_load_relaxed(ATOMIC_VAL) \
_Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_relaxed)
#define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \
_Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_relaxed)
DLLLOCAL static inline PyThreadState* _qore_PyRuntimeGILState_GetThreadState() {
return reinterpret_cast<PyThreadState*>(_Py_atomic_load_relaxed(&_PyRuntime.gilstate.tstate_current));
}
DLLLOCAL static inline void _qore_PyGILState_SetThisThreadState(PyThreadState* state) {
PyThread_tss_set(&_PyRuntime.gilstate.autoTSSkey, (void*)state);
}
DLLLOCAL static inline bool _qore_PyCeval_GetGilLockedStatus() {
return (bool)(_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked));
}
DLLLOCAL static inline PyThreadState* _qore_PyCeval_GetThreadState() {
return reinterpret_cast<PyThreadState*>(_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.last_holder));
}
DLLLOCAL static inline PyThreadState* _qore_PyCeval_SwapThreadState(PyThreadState* gil_state) {
PyThreadState* old = reinterpret_cast<PyThreadState*>(_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.last_holder));
if (old != gil_state) {
_Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder, (uintptr_t)gil_state);
}
return old;
}
#define _QORE_PYTHON_REENABLE_GIL_CHECK { assert(!_PyRuntime.gilstate.check_enabled); _PyRuntime.gilstate.check_enabled = 1; }
// In GIL-enabled mode, use standard PyThreadState_Swap
#define _QORE_PYTHREAD_STATE_SWAP(new_state) PyThreadState_Swap(new_state)
// gilstate_counter access macros - these are available in GIL-enabled Python
#define _QORE_GILSTATE_COUNTER_INC(tstate) (++(tstate)->gilstate_counter)
#define _QORE_GILSTATE_COUNTER_DEC(tstate) (--(tstate)->gilstate_counter)
#define _QORE_GILSTATE_COUNTER_GET(tstate) ((tstate)->gilstate_counter)
#define _QORE_GILSTATE_COUNTER_ASSERT_ONE(tstate) assert((tstate)->gilstate_counter == 1)
// Thread state management functions for GIL-enabled Python
DLLLOCAL static inline bool _qore_has_thread_state_attached() {
return PyGILState_Check();
}
DLLLOCAL static inline void _qore_acquire_thread_state(PyThreadState* tstate) {
PyEval_AcquireThread(tstate);
}
DLLLOCAL static inline void _qore_release_thread_state(PyThreadState* tstate) {
PyEval_ReleaseThread(tstate);
}
#endif