mod_info.name = QORE_PYTHON_MODULE_NAME;

mod_info.version = PACKAGE_VERSION;

mod_info.desc = "python module";

mod_info.author = "David Nichols";

mod_info.url = "http://qore.org";

mod_info.api_major = QORE_MODULE_API_MAJOR;

mod_info.api_minor = QORE_MODULE_API_MINOR;

mod_info.init = python_module_init;

mod_info.ns_init = python_module_ns_init;

mod_info.del = python_module_delete;

mod_info.parse_cmd = python_module_parse_cmd;

mod_info.license = QL_MIT;

mod_info.license_str = "MIT";

mod_info.info = new QoreHashNode(autoTypeInfo);

mod_info.info->setKeyValue("python_version", new QoreStringNodeMaker(PY_VERSION), nullptr);

mod_info.info->setKeyValue("python_major", PY_MAJOR_VERSION, nullptr);

mod_info.info->setKeyValue("python_minor", PY_MINOR_VERSION, nullptr);

mod_info.info->setKeyValue("python_micro", PY_MICRO_VERSION, nullptr);

qore_python_module_cmd_t cmd;

bool requires_arg = true;

DLLLOCAL qore_python_cmd_info_t(qore_python_module_cmd_t cmd, bool requires_arg)

: cmd(cmd), requires_arg(requires_arg) {

}

};

typedef std::map<std::string, qore_python_cmd_info_t> mcmap_t;

static mcmap_t mcmap = {

{"import", qore_python_cmd_info_t(py_mc_import, true)},

{"import-ns", qore_python_cmd_info_t(py_mc_import_ns, true)},

{"alias", qore_python_cmd_info_t(py_mc_alias, true)},

{"parse", qore_python_cmd_info_t(py_mc_parse, true)},

{"export-class", qore_python_cmd_info_t(py_mc_export_class, true)},

{"export-func", qore_python_cmd_info_t(py_mc_export_func, true)},

{"add-module-path", qore_python_cmd_info_t(py_mc_add_module_path, true)},

#if 0

{"reset-python", qore_python_cmd_info_t(py_mc_reset_python, false)},

#endif

};

static bool python_needs_shutdown = false;

static bool python_initialized = false;

bool python_shutdown = false;

int python_u_tld_key = -1;

int python_qobj_key = -1;

static sig_vec_t sig_vec = {

#ifndef _Q_WINDOWS

SIGSEGV, SIGBUS

#endif

QorePythonReferenceHolder mod(PyImport_ImportModule("sys"));

if (!mod) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "Python could not load module 'sys'");

}

// returns a borrowed reference

PyObject* mod_dict = PyModule_GetDict(*mod);

if (!mod_dict) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "Python module 'sys' has no dictionary");

}

// returns a borrowed reference

PyObject* value = PyDict_GetItemString(mod_dict, "version_info");

if (!value) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "symbol 'sys.version_info' not found; cannot verify the " \

"runtime version of the Python library");

}

if (!PyObject_HasAttrString(value, "major")) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "symbol 'sys.version.major' was not found; cannot " \

"verify the runtime version of the Python library");

}

QorePythonReferenceHolder py_major(PyObject_GetAttrString(value, "major"));

if (!PyLong_Check(*py_major)) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "symbol 'sys.version.major' has type '%s'; expecting " \

"'int'; cannot verify the runtime version of the Python library", Py_TYPE(*py_major)->tp_name);

}

long major = PyLong_AsLong(*py_major);

if (major != PY_MAJOR_VERSION) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "Python runtime major version is %ld, but the module was " \

"compiled with major version %d (%s)", major, PY_MAJOR_VERSION, PY_VERSION);

}

QorePythonReferenceHolder py_minor(PyObject_GetAttrString(value, "minor"));

if (!PyLong_Check(*py_minor)) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "symbol 'sys.version.minor' has type '%s'; expecting " \

"'int'; cannot verify the runtime version of the Python library", Py_TYPE(*py_minor)->tp_name);

}

long minor = PyLong_AsLong(*py_minor);

if (minor != PY_MINOR_VERSION) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "Python runtime version is %ld.%ld, but the module was " \

"compiled with version %d.%d (%s)", major, minor, PY_MAJOR_VERSION, PY_MINOR_VERSION, PY_VERSION);

}

//printd(5, "python runtime version OK: %ld.%ld.x =~ '%s'\n", major, minor, PY_VERSION);

if (python_initialized) {

_QORE_PYTHREAD_STATE_SWAP(nullptr);

_qore_acquire_thread_state(mainThreadState);

_qore_PyGILState_SetThisThreadState(mainThreadState);

}

python_shutdown = true;

if (python_needs_shutdown) {

int rc = Py_FinalizeEx();

if (rc) {

printd(0, "Unkown error shutting down Python: rc: %d\n", rc);

}

python_needs_shutdown = false;

}

if (!python_needs_shutdown) {

xsink->raiseException("PYTHON-RESET-ERROR", "The module was loaded into an existing Python process and " \

"therefore cannot be reset externally");

return -1;

}

unsigned cnt = QorePythonProgram::getProgramCount();

if (cnt) {

if (cnt <= 2) {

QoreProgram* pgm = getProgram();

if (pgm) {

QorePythonProgramData* pypgm = static_cast<QorePythonProgramData*>(pgm->removeExternalData(QORE_PYTHON_MODULE_NAME));

if (pypgm) {

pypgm->destructor(xsink);

pypgm->weakDeref();

if (*xsink) {

return -1;

}

--cnt;

}

}

}

if (cnt == 1 && qore_python_pgm) {

qore_python_pgm->destructor(xsink);

qore_python_pgm->weakDeref();

qore_python_pgm = nullptr;

--cnt;

}

if (cnt) {

xsink->raiseException("PYTHON-RESET-ERROR", "Cannot reset the Python library with %d Python program%s " \

"still valid", cnt, cnt == 1 ? "" : "s");

return -1;

}

}

python_module_shutdown();

SimpleRefHolder<QoreStringNode> err(python_module_init_intern(true));

if (err) {

xsink->raiseException("PYTHON-RESET-ERROR", err.release());

return -1;

}

return 0;

QoreStringNode* err = python_module_init_intern(false);

if (err) {

xsink.raiseException("MODULE-INIT-ERROR", err);

}

QoreProgram* pgm = getProgram();

assert(pgm->getRootNS() == rns);

if (!pgm->getExternalData(QORE_PYTHON_MODULE_NAME)) {

QoreNamespace* pyns = PNS->copy();

rns->addNamespace(pyns);

// NOTE: Use QoreProgramContextHelper instead of QoreExternalProgramContextHelper

// because QoreExternalProgramContextHelper uses runtime=true which triggers

// doTopLevelInstantiation(), setting tlpd->inst = true. This happens before the

// user's top-level local variables are defined (during %requires processing), so

// no variables are actually instantiated. When the helper destructs, tlpd->inst

// stays true even though no variables were instantiated. Later when runTopLevel()

// is called, it skips doTopLevelInstantiation because tlpd->inst is true, causing

// crashes when accessing top-level local variables defined after %requires python.

// QoreProgramContextHelper just sets the current program without triggering

// thread-local variable instantiation.

//

// Exception handling note: QoreProgramContextHelper doesn't use ExceptionSink because

// it only manages program context (save/restore), which doesn't throw. The

// QorePythonProgram constructor handles its own exceptions internally - any Python

// initialization errors are logged and handled within the constructor. This differs

// from QoreExternalProgramContextHelper which needed ExceptionSink for its runtime

// thread-local operations, not for the external data setup itself.

QoreProgramContextHelper pch(pgm);

pgm->setExternalData(QORE_PYTHON_MODULE_NAME, new QorePythonProgram(pgm, pyns));

}

#ifndef Py_GIL_DISABLED

#if PY_VERSION_HEX < 0x030C0000

// In Python 3.12+, PyGILState_Check() behavior changed - it returns 1 even after

// releasing the GIL because PyEval_ReleaseThread doesn't clear the TSS.

// In Python 3.13+, sub-interpreters also affect this behavior.

assert(!python_initialized || !PyGILState_Check());

#endif

// haveGil() uses our own tracking which should be accurate

assert(!python_initialized || !QorePythonProgram::haveGil());

#endif

if (qore_python_pgm) {

qore_python_pgm->doDeref();

qore_python_pgm = nullptr;

}

if (PNS) {

delete PNS;

PNS = nullptr;

}

python_module_shutdown();

//printd(5, "python_module_parse_cmd() cmd: '%s'\n", cmd.c_str());

const char* p = strchr(cmd.c_str(), ' ');

QoreString str;

QoreString arg;

if (p) {

QoreString nstr(&cmd, p - cmd.c_str());

str = nstr;

arg = cmd;

arg.replace(0, p - cmd.c_str() + 1, (const char*)nullptr);

arg.trim();

} else {

str = cmd;

str.trim();

}

mcmap_t::const_iterator i = mcmap.find(str.c_str());

if (i == mcmap.end()) {

QoreStringNode* desc = new QoreStringNodeMaker("unrecognized command '%s' in '%s' (valid commands: ", str.c_str(), cmd.c_str());

for (mcmap_t::const_iterator i = mcmap.begin(), e = mcmap.end(); i != e; ++i) {

if (i != mcmap.begin())

desc->concat(", ");

desc->sprintf("'%s'", i->first.c_str());

}

desc->concat(')');

xsink->raiseException("PYTHON-PARSE-COMMAND-ERROR", desc);

return;

}

if (i->second.requires_arg) {

if (arg.empty()) {

xsink->raiseException("PYTHON-PARSE-COMMAND-ERROR", "missing argument / command name in parse command: '%s'", cmd.c_str());

return;

}

} else {

if (!arg.empty()) {

xsink->raiseException("PYTHON-PARSE-COMMAND-ERROR", "extra argument / command name in parse command: '%s'", cmd.c_str());

return;

}

}

QoreProgram* pgm = getProgram();

QorePythonProgram* pypgm = static_cast<QorePythonProgram*>(pgm->getExternalData(QORE_PYTHON_MODULE_NAME));

//printd(5, "parse-cmd '%s' pypgm: %p pythonns: %p\n", arg.c_str(), pypgm, pypgm->getPythonNamespace());

if (!pypgm) {

QoreNamespace* pyns = PNS->copy();

pgm->getRootNS()->addNamespace(pyns);

pypgm = new QorePythonProgram(pgm, pyns);

pgm->setExternalData(QORE_PYTHON_MODULE_NAME, pypgm);

pgm->addFeature(QORE_PYTHON_MODULE_NAME);

}

i->second.cmd(xsink, arg, pypgm);

// process import statement

//printd(5, "py_mc_import() pypgm: %p arg: %s\n", pypgm, arg.c_str());

QorePythonHelper qph(pypgm, xsink);

if (qph.wasInterrupted()) {

return;

}

// see if there is a dot (.) in the name

qore_offset_t i = arg.find('.');

if (i < 0 || i == static_cast<qore_offset_t>(arg.size() - 1)) {

pypgm->import(xsink, arg.c_str());

return;

}

const char* symbol = arg.c_str() + i + 1;

arg.replaceChar(i, '\0');

arg.terminate(i);

if (!strcmp(symbol, "*")) {

pypgm->import(xsink, arg.c_str());

return;

}

pypgm->import(xsink, arg.c_str(), symbol);

// find end of qore namespace

qore_offset_t end = arg.find(' ');

if (end == -1) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "syntax: import-ns <qore-namespace> " \

"<python-module-path>: missing python module path argument; value given: '%s'", arg.c_str());

}

QoreString qore_ns(&arg, end);

QoreString py_mod_path(arg.c_str() + end + 1);

QoreProgram* pgm = getProgram();

if (!pgm) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "import-ns error: no current Program context");

}

QoreNamespace* ns = pgm->findNamespace(qore_ns);

if (!ns || ns == pgm->getRootNS()) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "import-ns error: Qore namespace '%s' not found",

qore_ns.c_str());

}

pypgm->importQoreNamespaceToPython(*ns, py_mod_path, xsink);

// find end of qore namespace

qore_offset_t end = arg.find(' ');

if (end == -1 || (size_t)end == (arg.size() - 1)) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "syntax: alias <python-source-path> " \

"<python-target-path: python target path argument; value given: '%s'", arg.c_str());

}

QoreString source_path(&arg, end);

QoreString target_path(arg.c_str() + end + 1);

pypgm->aliasDefinition(source_path, target_path);

// find end of qore namespace

qore_offset_t end = arg.find(' ');

if (end == -1 || (size_t)end == (arg.size() - 1)) {

throw QoreStandardException("PYTHON-MODULE-ERROR", "syntax: alias <python-source-path> " \

"<python-target-path: python target path argument; value given: '%s'", arg.c_str());

}

QoreString source_label(&arg, end);

QoreString source_code(arg.c_str() + end + 1);

ValueHolder val(pypgm->eval(xsink, source_code, source_label, Py_file_input, false), xsink);

QorePythonProgram* pypgm = static_cast<QorePythonProgram*>(pgm->getExternalData(QORE_PYTHON_MODULE_NAME));

if (!pypgm) {

QoreNamespace* pyns = PNS->copy();

pgm->getRootNS()->addNamespace(pyns);

pypgm = new QorePythonProgram(pgm, pyns);

pgm->setExternalData(QORE_PYTHON_MODULE_NAME, pypgm);

pgm->addFeature(QORE_PYTHON_MODULE_NAME);

}

// the following call adds the class to the current program as well

QorePythonHelper qph(pypgm, xsink);

if (qph.wasInterrupted()) {

return -1;

}

return pypgm->import(xsink, module, symbol);

: old_pgm(q_swap_thread_local_data(python_u_tld_key, (void*)pypgm)),

old_state(pypgm->setContext(xsink != nullptr)), new_pypgm(pypgm) {

//printd(5, "QorePythonHelper::QorePythonHelper() new: %p old: %p\n", pypgm, old_pgm);

if (xsink) {

if (wasInterrupted()) {

xsink->raiseException("PROGRAM-INTERRUPTED",

"program execution was interrupted while acquiring the Python GIL");

} else if (!old_state.valid && !new_pypgm->isValid()) {

// The interpreter has been deleted - raise an exception

xsink->raiseException("PYTHON-INTERPRETER-DELETED",

"cannot execute Python callable: the Python interpreter has been deleted");

}

}

if (!_qore_PyCeval_GetGilLockedStatus()) {

return false;

}

#if PY_VERSION_HEX < 0x030C0000

// In Python < 3.12, PyGILState_Check() is reliable. If it says we have the GIL,

// we have it even if t_state is NULL (can happen during early init before TSS is set).

if (t_state == nullptr && PyGILState_Check()) {

return true;

}

#endif

return _qore_PyCeval_GetThreadState() == t_state;

if (!_qore_PyCeval_GetGilLockedStatus()) {

return false;

}

#if PY_VERSION_HEX < 0x030C0000

// In Python < 3.12, PyGILState_Check() is reliable. If it says we have the GIL,

// we have it even if both states are NULL (can happen during early init).

if (state0 == nullptr && state1 == nullptr && PyGILState_Check()) {

return true;

}

#endif

PyThreadState* gs = _qore_PyCeval_GetThreadState();

return gs == state0 || gs == state1;

: new_thread_state(new_thread_state), state(nullptr), t_state(nullptr), release_gil(true) {

// CRITICAL: Check if Python is shutting down or not initialized before ANY Python API calls.

// After Py_FinalizeEx(), calling PyGILState_Ensure() will crash because the TSS points

// to a stale thread state with a freed interpreter.

if (python_shutdown || !Py_IsInitialized()) {

printd(5, "QorePythonGilHelper ctor: Python is shutting down, skipping initialization\n");

return;

}

assert(new_thread_state);

// Now safe to call Python APIs

state = _qore_PyRuntimeGILState_GetThreadState();

t_state = PyGILState_GetThisThreadState();

release_gil = !_qore_has_gil(t_state, new_thread_state);

#ifdef Py_GIL_DISABLED

// In free-threading mode, use PyGILState_Ensure to properly

// initialize the thread and ensure a valid thread state is attached.

gstate = PyGILState_Ensure();

// Now we have a thread state attached

t_state = PyGILState_GetThisThreadState();

if (t_state != new_thread_state) {

PyThreadState_Swap(new_thread_state);

}

_qore_PyGILState_SetThisThreadState(new_thread_state);

_QORE_GILSTATE_COUNTER_INC(new_thread_state);

#elif PY_VERSION_HEX >= 0x030D0000

// Python 3.13+ GIL mode - use PyEval_AcquireThread/ReleaseThread which properly

// handle TSS and fast TLS synchronization

if (release_gil) {

// If bound_gilstate is stale, clear it to avoid tstate_activate assertions.

if (qore_py_get_bound_gilstate(new_thread_state)

&& PyGILState_GetThisThreadState() != new_thread_state) {

qore_py_set_bound_gilstate(new_thread_state, 0);

}

// Need to acquire the GIL with our specific thread state

// CRITICAL: If there's a stale TSS from a previous interpreter, we must clear it first.

// PyEval_AcquireThread -> _PyThreadState_Attach tries to detach any existing thread state,

// and if the TSS doesn't match, it will assert.

PyEval_AcquireThread(new_thread_state);

} else {

// Already have the GIL - swap to our thread state if needed

if (t_state != new_thread_state) {

printd(5, "QorePythonGilHelper ctor: swapping from %p to %p\n", t_state, new_thread_state);

PyThreadState_Swap(new_thread_state);

}

}

// CRITICAL: Update our tracking AFTER acquiring the GIL.

// _qore_gil_held must be set for _qore_PyCeval_GetGilLockedStatus() to return true.

_qore_gil_held = true;

_QORE_GILSTATE_COUNTER_INC(new_thread_state);

_qore_PyGILState_SetThisThreadState(new_thread_state);

#else

if (release_gil) {

#if PY_VERSION_HEX >= 0x030C0000

// Ensure bound_gilstate and TSS are consistent before acquiring the GIL in Python 3.12.

PyThreadState* tss_before = PyGILState_GetThisThreadState();

if (qore_py_get_bound_gilstate(new_thread_state) && tss_before != new_thread_state) {

_qore_PyGILState_SetTSS(new_thread_state);

PyThreadState* tss_after = PyGILState_GetThisThreadState();

if (tss_after != new_thread_state) {

_qore_PyGILState_ClearTSS();

qore_py_set_bound_gilstate(new_thread_state, 0);

} else {

qore_py_set_bound_gilstate(new_thread_state, 1);

}

}

#endif

_qore_acquire_thread_state(new_thread_state);

// Use _qore_tss_tstate for assertion since Python's TSS (PyGILState_GetThisThreadState)

// might be stale from a deleted interpreter in Python 3.12

assert(_qore_tss_tstate == new_thread_state);

} else {

// NOTE: In Python 3.12, t_state (from PyGILState_GetThisThreadState()) may be stale

// after PyEval_ReleaseThread() since Python's TSS isn't cleared.

// _qore_has_gil() verified we have the GIL with either t_state or new_thread_state,

// so check against both possibilities.

PyThreadState* ceval_ts = _qore_PyCeval_GetThreadState();

assert(ceval_ts == t_state || ceval_ts == new_thread_state);

}

// NOTE: even if the current thread state is equal to the new one, we still need to set all thread states in all

// locations

_QORE_GILSTATE_COUNTER_INC(new_thread_state);

_QORE_PYTHREAD_STATE_SWAP(new_thread_state);

// set this thread state

_qore_PyGILState_SetThisThreadState(new_thread_state);

#if PY_VERSION_HEX < 0x030C0000

// NOTE: In Python 3.12+, PyGILState_GetThisThreadState() can be unreliable because

// PyEval_ReleaseThread() doesn't clear the autoTSSkey. Our tracking is authoritative.

assert(PyGILState_GetThisThreadState() == new_thread_state);

assert(PyGILState_Check());

#else

// Python 3.12+: PyGILState_Check() and PyGILState_GetThisThreadState() rely on Python's

// internal TSS (autoTSSkey) which can be stale after PyEval_ReleaseThread() or corrupted

// by external modules like JNI. Our thread-local tracking (_qore_tss_tstate) is reliable.

assert(_qore_PyCeval_GetGilLockedStatus());

#endif

#endif

initialized = true;

// Prepare for sub-interpreter creation in free-threading mode.

// In free-threading mode with mimalloc, each thread state has thread-local heap data.

// PyGILState_Ensure() set up heap data for the main interpreter. Before creating a

// sub-interpreter, we need to detach so Py_NewInterpreterFromConfig can properly

// initialize heap data for the new interpreter.

// Swap to NULL thread state to detach from main interpreter's thread state.

// This allows Py_NewInterpreterFromConfig to properly attach a new thread state

// with correctly initialized mimalloc heap for the sub-interpreter.

PyThreadState_Swap(nullptr);

gstate_released = true;

// This function is called after Py_NewInterpreter() to switch from the main interpreter's

// thread state (which was used to acquire the GIL in the constructor) to the new sub-interpreter's

// thread state.

//

// GIL State Counter Balancing:

// - Constructor incremented the MAIN interpreter's gilstate_counter

// - Destructor will decrement new_thread_state's gilstate_counter

// - Since we're switching new_thread_state to point to other_state (sub-interpreter),

// we must:

// 1. Decrement the MAIN interpreter's counter to balance the constructor's increment

// 2. Increment the SUB interpreter's counter to balance the destructor's decrement

//

// Without step 1, the main thread state's counter would leak (+1 for each sub-interpreter

// creation), potentially breaking later GIL bookkeeping assertions.

// Without step 2, Py_NewInterpreter's thread state (which starts with gilstate_counter=1)

// would be decremented to 0, causing PyEval_RestoreThread to fail its assertion.

//

// See design/python-module.md for detailed explanation.

#ifdef Py_GIL_DISABLED

// In free-threading mode, Py_NewInterpreterFromConfig already attached the new thread state.

// Just update our tracking variable so the destructor restores correctly.

// Do NOT call PyThreadState_Swap again as it may corrupt the thread's heap state.

// gilstate_counter balancing is not needed in free-threading mode.

new_thread_state = other_state;

#else

assert(_qore_PyCeval_GetGilLockedStatus() && _qore_PyCeval_GetThreadState());

// CRITICAL: Decrement the main thread state's counter to balance the increment in the

// constructor. This must be done BEFORE changing new_thread_state.

_QORE_GILSTATE_COUNTER_DEC(new_thread_state);

// Update new_thread_state so the destructor releases the correct thread state

new_thread_state = other_state;

// CRITICAL: Increment the new thread state's gilstate_counter to balance the decrement

// in the destructor.

_QORE_GILSTATE_COUNTER_INC(other_state);

_QORE_PYTHREAD_STATE_SWAP(other_state);

_qore_PyCeval_SwapThreadState(other_state);

_qore_PyGILState_SetThisThreadState(other_state);

#endif