/* indent-tabs-mode: nil -*- */
/*
python Qore module
Copyright (C) 2020 - 2026 Qore Technologies, s.r.o.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "python-module.h"
#include "QC_PythonProgram.h"
#include "QorePythonProgram.h"
#include "QorePythonStackLocationHelper.h"
#include
static void python_module_init(QoreModuleInitContext& ctx, ExceptionSink& xsink);
static void python_module_ns_init(QoreNamespace* rns, QoreNamespace* qns, ExceptionSink& xsink);
static void python_module_delete();
static void python_module_parse_cmd(const QoreString& cmd, ExceptionSink* xsink);
static QoreStringNode* python_module_init_intern(bool repeat);
// module declaration for Qore 0.9.5+
void python_qore_module_desc(QoreModuleInfo& mod_info) {
mod_info.name = QORE_PYTHON_MODULE_NAME;
mod_info.version = PACKAGE_VERSION;
mod_info.desc = "python module";
mod_info.author = "David Nichols";
mod_info.url = "http://qore.org";
mod_info.api_major = QORE_MODULE_API_MAJOR;
mod_info.api_minor = QORE_MODULE_API_MINOR;
mod_info.init = python_module_init;
mod_info.ns_init = python_module_ns_init;
mod_info.del = python_module_delete;
mod_info.parse_cmd = python_module_parse_cmd;
mod_info.license = QL_MIT;
mod_info.license_str = "MIT";
mod_info.info = new QoreHashNode(autoTypeInfo);
mod_info.info->setKeyValue("python_version", new QoreStringNodeMaker(PY_VERSION), nullptr);
mod_info.info->setKeyValue("python_major", PY_MAJOR_VERSION, nullptr);
mod_info.info->setKeyValue("python_minor", PY_MINOR_VERSION, nullptr);
mod_info.info->setKeyValue("python_micro", PY_MICRO_VERSION, nullptr);
}
QoreNamespace* PNS = nullptr;
PyThreadState* mainThreadState = nullptr;
QorePythonClass* QC_PYTHONBASEOBJECT;
qore_classid_t CID_PYTHONBASEOBJECT;
// module cmd type
using qore_python_module_cmd_t = void (*) (ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
static void py_mc_import(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
static void py_mc_import_ns(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
static void py_mc_alias(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
static void py_mc_parse(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
static void py_mc_export_class(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
static void py_mc_export_func(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
static void py_mc_add_module_path(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
//static void py_mc_reset_python(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm);
struct qore_python_cmd_info_t {
qore_python_module_cmd_t cmd;
bool requires_arg = true;
DLLLOCAL qore_python_cmd_info_t(qore_python_module_cmd_t cmd, bool requires_arg)
: cmd(cmd), requires_arg(requires_arg) {
}
};
// module cmds
typedef std::map<:string qore_python_cmd_info_t> mcmap_t;
static mcmap_t mcmap = {
{"import", qore_python_cmd_info_t(py_mc_import, true)},
{"import-ns", qore_python_cmd_info_t(py_mc_import_ns, true)},
{"alias", qore_python_cmd_info_t(py_mc_alias, true)},
{"parse", qore_python_cmd_info_t(py_mc_parse, true)},
{"export-class", qore_python_cmd_info_t(py_mc_export_class, true)},
{"export-func", qore_python_cmd_info_t(py_mc_export_func, true)},
{"add-module-path", qore_python_cmd_info_t(py_mc_add_module_path, true)},
#if 0
{"reset-python", qore_python_cmd_info_t(py_mc_reset_python, false)},
#endif
};
static bool python_needs_shutdown = false;
static bool python_initialized = false;
bool python_shutdown = false;
int python_u_tld_key = -1;
int python_qobj_key = -1;
static sig_vec_t sig_vec = {
#ifndef _Q_WINDOWS
SIGSEGV, SIGBUS
#endif
};
static void check_python_version() {
QorePythonReferenceHolder mod(PyImport_ImportModule("sys"));
if (!mod) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "Python could not load module 'sys'");
}
// returns a borrowed reference
PyObject* mod_dict = PyModule_GetDict(*mod);
if (!mod_dict) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "Python module 'sys' has no dictionary");
}
// returns a borrowed reference
PyObject* value = PyDict_GetItemString(mod_dict, "version_info");
if (!value) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "symbol 'sys.version_info' not found; cannot verify the " \
"runtime version of the Python library");
}
if (!PyObject_HasAttrString(value, "major")) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "symbol 'sys.version.major' was not found; cannot " \
"verify the runtime version of the Python library");
}
QorePythonReferenceHolder py_major(PyObject_GetAttrString(value, "major"));
if (!PyLong_Check(*py_major)) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "symbol 'sys.version.major' has type '%s'; expecting " \
"'int'; cannot verify the runtime version of the Python library", Py_TYPE(*py_major)->tp_name);
}
long major = PyLong_AsLong(*py_major);
if (major != PY_MAJOR_VERSION) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "Python runtime major version is %ld, but the module was " \
"compiled with major version %d (%s)", major, PY_MAJOR_VERSION, PY_VERSION);
}
QorePythonReferenceHolder py_minor(PyObject_GetAttrString(value, "minor"));
if (!PyLong_Check(*py_minor)) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "symbol 'sys.version.minor' has type '%s'; expecting " \
"'int'; cannot verify the runtime version of the Python library", Py_TYPE(*py_minor)->tp_name);
}
long minor = PyLong_AsLong(*py_minor);
if (minor != PY_MINOR_VERSION) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "Python runtime version is %ld.%ld, but the module was " \
"compiled with version %d.%d (%s)", major, minor, PY_MAJOR_VERSION, PY_MINOR_VERSION, PY_VERSION);
}
//printd(5, "python runtime version OK: %ld.%ld.x =~ '%s'\n", major, minor, PY_VERSION);
}
static void python_module_shutdown() {
if (python_initialized) {
_QORE_PYTHREAD_STATE_SWAP(nullptr);
_qore_acquire_thread_state(mainThreadState);
_qore_PyGILState_SetThisThreadState(mainThreadState);
}
python_shutdown = true;
if (python_needs_shutdown) {
int rc = Py_FinalizeEx();
if (rc) {
printd(0, "Unkown error shutting down Python: rc: %d\n", rc);
}
python_needs_shutdown = false;
}
}
#if 0
// does not work with modules like tensorflow that do not unload cleanly
int q_reset_python(ExceptionSink* xsink) {
if (!python_needs_shutdown) {
xsink->raiseException("PYTHON-RESET-ERROR", "The module was loaded into an existing Python process and " \
"therefore cannot be reset externally");
return -1;
}
unsigned cnt = QorePythonProgram::getProgramCount();
if (cnt) {
if (cnt <= 2) {
QoreProgram* pgm = getProgram();
if (pgm) {
QorePythonProgramData* pypgm = static_cast(pgm->removeExternalData(QORE_PYTHON_MODULE_NAME));
if (pypgm) {
pypgm->destructor(xsink);
pypgm->weakDeref();
if (*xsink) {
return -1;
}
--cnt;
}
}
}
if (cnt == 1 && qore_python_pgm) {
qore_python_pgm->destructor(xsink);
qore_python_pgm->weakDeref();
qore_python_pgm = nullptr;
--cnt;
}
if (cnt) {
xsink->raiseException("PYTHON-RESET-ERROR", "Cannot reset the Python library with %d Python program%s " \
"still valid", cnt, cnt == 1 ? "" : "s");
return -1;
}
}
python_module_shutdown();
SimpleRefHolder err(python_module_init_intern(true));
if (err) {
xsink->raiseException("PYTHON-RESET-ERROR", err.release());
return -1;
}
return 0;
}
#endif
static void python_module_init(QoreModuleInitContext& ctx, ExceptionSink& xsink) {
QoreStringNode* err = python_module_init_intern(false);
if (err) {
xsink.raiseException("MODULE-INIT-ERROR", err);
}
}
// defined in the generated ql_python.cpp (from ql_python.qpp)
DLLLOCAL void init_python_functions(QoreNamespace& ns);
static QoreStringNode* python_module_init_intern(bool repeat) {
if (!PNS) {
PNS = new QoreNamespace("Python");
PNS->addSystemClass(initPythonProgramClass(*PNS));
// register Python namespace functions (e.g. Python::set_save_object_callback())
init_python_functions(*PNS);
QC_PYTHONBASEOBJECT = new QorePythonClass("__qore_base__", "::Python::__qore_base__");
CID_PYTHONBASEOBJECT = QC_PYTHONBASEOBJECT->getID();
PNS->addSystemClass(QC_PYTHONBASEOBJECT->copy());
// Add constant to indicate if this is a free-threading Python build
#ifdef Py_GIL_DISABLED
PNS->addConstant("FreeThreading", true);
#else
PNS->addConstant("FreeThreading", false);
#endif
}
if (!repeat) {
python_u_tld_key = q_get_unique_thread_local_data_key();
python_qobj_key = q_get_unique_thread_local_data_key();
}
// initialize python library; do not register signal handlers
if (!Py_IsInitialized()) {
if (PyImport_AppendInittab("qoreloader", PyInit_qoreloader) == -1) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "cannot append the qoreloader module to Python");
}
Py_InitializeEx(0);
#ifdef QORE_ALLOW_PYTHON_SHUTDOWN
// issue# 4290: if we actively shut down Python on exit, then exit handlers in modules
// (such as the h5py module in version 3.3.0) will cause a crash when the process exits,
// as it requires the Python library to be still in place and initialized
python_needs_shutdown = true;
#endif
python_initialized = true;
//printd(5, "python_module_init() Python initialized\n");
}
if (!repeat) {
#ifndef _Q_WINDOWS
sig_vec_t new_sig_vec;
for (int sig : sig_vec) {
QoreStringNode *err = qore_reassign_signal(sig, QORE_PYTHON_MODULE_NAME);
if (err) {
// ignore errors; already assigned to another module
err->deref();
}
new_sig_vec.push_back(sig);
}
if (!new_sig_vec.empty()) {
sigset_t mask;
// setup signal mask
sigemptyset(&mask);
for (auto& sig : new_sig_vec) {
//printd(LogLevel, "python_module_init() unblocking signal %d\n", sig);
sigaddset(&mask, sig);
}
// unblock threads
pthread_sigmask(SIG_UNBLOCK, &mask, 0);
}
#endif
}
// ensure that runtime version matches compiled version
check_python_version();
// Ensure sys.path is initialized when embedding (esp. for debug builds).
const char* pyhome = getenv("PYTHONHOME");
const char* pypath = getenv("PYTHONPATH");
if ((pyhome && *pyhome) || (pypath && *pypath)) {
#ifdef _Q_WINDOWS
const char path_sep = ';';
#else
const char path_sep = ':';
#endif
std::string path;
if (pypath) {
path += pypath;
}
if (pyhome && *pyhome) {
if (!path.empty()) {
path += path_sep;
}
path += pyhome;
path += "/Lib";
path += path_sep;
path += pyhome;
path += "/Modules";
}
PyObject* sys_path = PySys_GetObject("path"); // borrowed
if (!sys_path || !PyList_Check(sys_path)) {
sys_path = PyList_New(0);
if (sys_path) {
PySys_SetObject("path", sys_path);
Py_DECREF(sys_path);
}
}
if (sys_path && PyList_Check(sys_path)) {
size_t start = 0;
// Preserve empty entries to keep CWD semantics (ex: leading/trailing separators).
while (start <= path.size()) {
size_t end = path.find(path_sep, start);
if (end == std::string::npos) {
end = path.size();
}
std::string entry = path.substr(start, end - start);
PyObject* py_entry = PyUnicode_DecodeFSDefault(entry.c_str());
if (py_entry) {
PyList_Append(sys_path, py_entry);
Py_DECREF(py_entry);
}
start = end + 1;
}
}
}
// Initialize thread-local state tracking to match Python's state
// This must be done before creating any QorePythonProgram instances
#ifdef Py_GIL_DISABLED
// In free-threading mode, ensure main thread state is attached before any Python API calls
mainThreadState = _qore_safe_thread_state_get();
//printd(5, "python_module_init_intern() mainThreadState: %p current: %p\n",
// mainThreadState, PyGILState_GetThisThreadState());
if (!PyGILState_GetThisThreadState()) {
PyThreadState_Swap(mainThreadState);
}
#else
// In GIL mode, PyGILState_GetThisThreadState() might return NULL during early init
// even though we have the GIL. Use _qore_safe_thread_state_get() which tolerates missing TSS.
PyThreadState* init_tstate = PyGILState_GetThisThreadState();
if (!init_tstate) {
// TSS not set up yet - get the actual thread state and set it up
init_tstate = PyThreadState_Get();
// Also update mainThreadState for later use
mainThreadState = init_tstate;
}
_qore_PyGILState_SetThisThreadState(init_tstate);
// We have the GIL at this point after Py_InitializeEx()
_qore_gil_held = true;
#endif
if (init_global_qore_python_pgm()) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "failed to initialize \"python\" module");
}
#ifdef Py_GIL_DISABLED
// In free-threading mode, use PyGILState_Ensure to properly set up the thread for Python ops
// This ensures the mimalloc heap is properly initialized for this thread
PyGILState_STATE gstate = PyGILState_Ensure();
//printd(5, "python_module_init_intern() after PyGILState_Ensure: current: %p gstate: %d\n",
// PyGILState_GetThisThreadState(), (int)gstate);
#endif
if (QorePythonProgram::staticInit() || QorePythonStackLocationHelper::staticInit()) {
#ifdef Py_GIL_DISABLED
PyGILState_Release(gstate);
#endif
throw QoreStandardException("PYTHON-MODULE-ERROR", "failed to initialize \"python\" module");
}
#ifdef Py_GIL_DISABLED
PyGILState_Release(gstate);
//printd(5, "python_module_init_intern() after PyGILState_Release: current: %p\n",
// PyGILState_GetThisThreadState());
#endif
#ifndef Py_GIL_DISABLED
mainThreadState = PyThreadState_Get();
if (python_initialized) {
#if PY_VERSION_HEX >= 0x030D0000
// Python 3.13+ changed thread state management significantly
// Use PyEval_ReleaseThread which properly clears both TSS and fast TLS
printd(5, "python_module_init: before release, mainThreadState: %p TSS: %p GIL: %d\n",
mainThreadState, PyGILState_GetThisThreadState(), PyGILState_Check());
PyEval_ReleaseThread(mainThreadState);
printd(5, "python_module_init: after release, TSS: %p GIL: %d\n",
PyGILState_GetThisThreadState(), PyGILState_Check());
_qore_PyGILState_SetThisThreadState(nullptr);
#else
// release the current thread state after initialization
_qore_release_thread_state(mainThreadState);
// Our tracking should be cleared by _qore_release_thread_state
assert(!_qore_PyRuntimeGILState_GetThreadState());
_qore_PyGILState_SetThisThreadState(nullptr);
// NOTE: In Python 3.12, PyEval_ReleaseThread does NOT clear PyGILState_GetThisThreadState()
// because it doesn't update the autoTSSkey. This is different from earlier Python versions.
// We only check haveGil() which uses our own tracking.
assert(!QorePythonProgram::haveGil());
#endif
}
#else
// In free-threading mode, don't release the thread state after initialization
// We keep the main thread state attached for Python operations
#endif
if (!repeat) {
tclist.push(QorePythonProgram::pythonThreadCleanup, nullptr);
}
return nullptr;
}
static void python_module_ns_init(QoreNamespace* rns, QoreNamespace* qns, ExceptionSink& xsink) {
QoreProgram* pgm = getProgram();
assert(pgm->getRootNS() == rns);
if (!pgm->getExternalData(QORE_PYTHON_MODULE_NAME)) {
QoreNamespace* pyns = PNS->copy();
rns->addNamespace(pyns);
// NOTE: Use QoreProgramContextHelper instead of QoreExternalProgramContextHelper
// because QoreExternalProgramContextHelper uses runtime=true which triggers
// doTopLevelInstantiation(), setting tlpd->inst = true. This happens before the
// user's top-level local variables are defined (during %requires processing), so
// no variables are actually instantiated. When the helper destructs, tlpd->inst
// stays true even though no variables were instantiated. Later when runTopLevel()
// is called, it skips doTopLevelInstantiation because tlpd->inst is true, causing
// crashes when accessing top-level local variables defined after %requires python.
// QoreProgramContextHelper just sets the current program without triggering
// thread-local variable instantiation.
//
// Exception handling note: QoreProgramContextHelper doesn't use ExceptionSink because
// it only manages program context (save/restore), which doesn't throw. The
// QorePythonProgram constructor handles its own exceptions internally - any Python
// initialization errors are logged and handled within the constructor. This differs
// from QoreExternalProgramContextHelper which needed ExceptionSink for its runtime
// thread-local operations, not for the external data setup itself.
QoreProgramContextHelper pch(pgm);
pgm->setExternalData(QORE_PYTHON_MODULE_NAME, new QorePythonProgram(pgm, pyns));
}
#ifndef Py_GIL_DISABLED
#if PY_VERSION_HEX < 0x030C0000
// In Python 3.12+, PyGILState_Check() behavior changed - it returns 1 even after
// releasing the GIL because PyEval_ReleaseThread doesn't clear the TSS.
// In Python 3.13+, sub-interpreters also affect this behavior.
assert(!python_initialized || !PyGILState_Check());
#endif
// haveGil() uses our own tracking which should be accurate
assert(!python_initialized || !QorePythonProgram::haveGil());
#endif
}
static void python_module_delete() {
if (qore_python_pgm) {
qore_python_pgm->doDeref();
qore_python_pgm = nullptr;
}
if (PNS) {
delete PNS;
PNS = nullptr;
}
python_module_shutdown();
}
static void python_module_parse_cmd(const QoreString& cmd, ExceptionSink* xsink) {
//printd(5, "python_module_parse_cmd() cmd: '%s'\n", cmd.c_str());
const char* p = strchr(cmd.c_str(), ' ');
QoreString str;
QoreString arg;
if (p) {
QoreString nstr(&cmd, p - cmd.c_str());
str = nstr;
arg = cmd;
arg.replace(0, p - cmd.c_str() + 1, (const char*)nullptr);
arg.trim();
} else {
str = cmd;
str.trim();
}
mcmap_t::const_iterator i = mcmap.find(str.c_str());
if (i == mcmap.end()) {
QoreStringNode* desc = new QoreStringNodeMaker("unrecognized command '%s' in '%s' (valid commands: ", str.c_str(), cmd.c_str());
for (mcmap_t::const_iterator i = mcmap.begin(), e = mcmap.end(); i != e; ++i) {
if (i != mcmap.begin())
desc->concat(", ");
desc->sprintf("'%s'", i->first.c_str());
}
desc->concat(')');
xsink->raiseException("PYTHON-PARSE-COMMAND-ERROR", desc);
return;
}
if (i->second.requires_arg) {
if (arg.empty()) {
xsink->raiseException("PYTHON-PARSE-COMMAND-ERROR", "missing argument / command name in parse command: '%s'", cmd.c_str());
return;
}
} else {
if (!arg.empty()) {
xsink->raiseException("PYTHON-PARSE-COMMAND-ERROR", "extra argument / command name in parse command: '%s'", cmd.c_str());
return;
}
}
QoreProgram* pgm = getProgram();
QorePythonProgram* pypgm = static_cast(pgm->getExternalData(QORE_PYTHON_MODULE_NAME));
//printd(5, "parse-cmd '%s' pypgm: %p pythonns: %p\n", arg.c_str(), pypgm, pypgm->getPythonNamespace());
if (!pypgm) {
QoreNamespace* pyns = PNS->copy();
pgm->getRootNS()->addNamespace(pyns);
pypgm = new QorePythonProgram(pgm, pyns);
pgm->setExternalData(QORE_PYTHON_MODULE_NAME, pypgm);
pgm->addFeature(QORE_PYTHON_MODULE_NAME);
}
i->second.cmd(xsink, arg, pypgm);
}
// %module-cmd(python) import
static void py_mc_import(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm) {
// process import statement
//printd(5, "py_mc_import() pypgm: %p arg: %s\n", pypgm, arg.c_str());
QorePythonHelper qph(pypgm, xsink);
if (qph.wasInterrupted()) {
return;
}
// see if there is a dot (.) in the name
qore_offset_t i = arg.find('.');
if (i < 0 || i == static_cast(arg.size() - 1)) {
pypgm->import(xsink, arg.c_str());
return;
}
const char* symbol = arg.c_str() + i + 1;
arg.replaceChar(i, '\0');
arg.terminate(i);
if (!strcmp(symbol, "*")) {
pypgm->import(xsink, arg.c_str());
return;
}
pypgm->import(xsink, arg.c_str(), symbol);
}
// %module-cmd(python) import-ns
static void py_mc_import_ns(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm) {
// find end of qore namespace
qore_offset_t end = arg.find(' ');
if (end == -1) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "syntax: import-ns " \
": missing python module path argument; value given: '%s'", arg.c_str());
}
QoreString qore_ns(&arg, end);
QoreString py_mod_path(arg.c_str() + end + 1);
QoreProgram* pgm = getProgram();
if (!pgm) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "import-ns error: no current Program context");
}
QoreNamespace* ns = pgm->findNamespace(qore_ns);
if (!ns || ns == pgm->getRootNS()) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "import-ns error: Qore namespace '%s' not found",
qore_ns.c_str());
}
pypgm->importQoreNamespaceToPython(*ns, py_mod_path, xsink);
}
// %module-cmd(python) alias
static void py_mc_alias(ExceptionSink* xsink, QoreString& arg, QorePythonProgram* pypgm) {
// find end of qore namespace
qore_offset_t end = arg.find(' ');
if (end == -1 || (size_t)end == (arg.size() - 1)) {
throw QoreStandardException("PYTHON-MODULE-ERROR", "syntax: alias " \
"aliasDefinition(source_path, target_path);
}
// %module-cmd(python) parse