Refactor Python

This commit is contained in:
Lord-Grey 2024-11-14 23:00:10 +01:00
parent 1dfbdb7f6f
commit 60f5a07500
8 changed files with 565 additions and 412 deletions

View File

@ -74,7 +74,7 @@ signals:
void setInputImage(int priority, const Image<ColorRgb>& image, int timeout_ms, bool clearEffect);
private:
void setModuleParameters();
bool setModuleParameters();
void addImage();
Hyperion* _hyperion;

View File

@ -13,12 +13,6 @@ class EffectModule: public QObject
Q_OBJECT
public:
// Python 3 module def
static struct PyModuleDef moduleDef;
// Init module
static PyObject* PyInit_hyperion();
// Register module once
static void registerHyperionExtensionModule();

View File

@ -1,5 +1,9 @@
#pragma once
#undef slots
#include <Python.h>
#define slots Q_SLOTS
///
/// @brief Handle the PythonInit, module registers and DeInit
///
@ -10,4 +14,6 @@ private:
PythonInit();
~PythonInit();
void handlePythonError(PyStatus status, PyConfig& config);
};

View File

@ -9,6 +9,8 @@
#include "Python.h"
#define slots
#include <python/PythonUtils.h>
class Logger;
class PythonProgram
@ -17,9 +19,15 @@ public:
PythonProgram(const QString & name, Logger * log);
~PythonProgram();
operator PyThreadState* ()
{
return _tstate;
}
void execute(const QByteArray &python_code);
private:
QString _name;
Logger* _log;
PyThreadState* _tstate;

View File

@ -66,36 +66,76 @@ int Effect::getRemaining() const
return timeout;
}
void Effect::setModuleParameters()
bool Effect::setModuleParameters()
{
// import the buildtin Hyperion module
PyObject* module = PyImport_ImportModule("hyperion");
// add a capsule containing 'this' to the module to be able to retrieve the effect from the callback function
PyModule_AddObject(module, "__effectObj", PyCapsule_New((void*)this, "hyperion.__effectObj", nullptr));
if (module == nullptr) {
PyErr_Print(); // Print error if module import fails
return false;
}
// add ledCount variable to the interpreter
// Add a capsule containing 'this' to the module for callback access
PyObject* capsule = PyCapsule_New((void*)this, "hyperion.__effectObj", nullptr);
if (capsule == nullptr || PyModule_AddObject(module, "__effectObj", capsule) < 0) {
PyErr_Print(); // Print error if adding capsule fails
Py_XDECREF(module); // Clean up if capsule addition fails
Py_XDECREF(capsule);
return false;
}
// Add ledCount variable to the interpreter
int ledCount = 0;
QMetaObject::invokeMethod(_hyperion, "getLedCount", Qt::BlockingQueuedConnection, Q_RETURN_ARG(int, ledCount));
PyObject_SetAttrString(module, "ledCount", Py_BuildValue("i", ledCount));
PyObject* ledCountObj = Py_BuildValue("i", ledCount);
if (PyObject_SetAttrString(module, "ledCount", ledCountObj) < 0) {
PyErr_Print(); // Print error if setting attribute fails
}
Py_XDECREF(ledCountObj);
// add minimumWriteTime variable to the interpreter
// Add minimumWriteTime variable to the interpreter
int latchTime = 0;
QMetaObject::invokeMethod(_hyperion, "getLatchTime", Qt::BlockingQueuedConnection, Q_RETURN_ARG(int, latchTime));
PyObject_SetAttrString(module, "latchTime", Py_BuildValue("i", latchTime));
PyObject* latchTimeObj = Py_BuildValue("i", latchTime);
if (PyObject_SetAttrString(module, "latchTime", latchTimeObj) < 0) {
PyErr_Print(); // Print error if setting attribute fails
}
Py_XDECREF(latchTimeObj);
// add a args variable to the interpreter
PyObject_SetAttrString(module, "args", EffectModule::json2python(_args));
// Add args variable to the interpreter
PyObject* argsObj = EffectModule::json2python(_args);
if (PyObject_SetAttrString(module, "args", argsObj) < 0) {
PyErr_Print(); // Print error if setting attribute fails
}
Py_XDECREF(argsObj);
// decref the module
// Decrement module reference
Py_XDECREF(module);
return true;
}
void Effect::run()
{
PythonProgram program(_name, _log);
setModuleParameters();
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState* prev_thread_state = PyThreadState_Swap(program);
#endif
if (!setModuleParameters())
{
Error(_log, "Failed to set Module parameters. Effect will not be executed.");
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(prev_thread_state);
#endif
return;
}
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(prev_thread_state);
#endif
// Set the end time if applicable
if (_timeout > 0)

View File

@ -17,25 +17,81 @@
#include <QNetworkAccessManager>
#include <QEventLoop>
// Define a struct for per-interpreter state
typedef struct {
} hyperion_module_state;
// Macro to access the module state
#define GET_HYPERION_STATE(module) ((hyperion_module_state*)PyModule_GetState(module))
// Get the effect from the capsule
#define getEffect() static_cast<Effect*>((Effect*)PyCapsule_Import("hyperion.__effectObj", 0))
// create the hyperion module
struct PyModuleDef EffectModule::moduleDef = {
PyModuleDef_HEAD_INIT,
"hyperion", /* m_name */
"Hyperion module", /* m_doc */
-1, /* m_size */
EffectModule::effectMethods, /* m_methods */
NULL, /* m_reload */
NULL, /* m_traverse */
NULL, /* m_clear */
NULL, /* m_free */
// Module execution function for multi-phase init
static int hyperion_exec(PyObject* module) {
// Initialize per-interpreter state
hyperion_module_state* state = GET_HYPERION_STATE(module);
if (state == NULL)
{
return -1;
}
return 0;
}
// Module creation function for multi-phase init, used in Py_mod_create slot
static PyObject* hyperion_create(PyModuleDef* def, PyObject* args) {
PyObject* module = PyModule_Create(def);
if (!module)
{
return NULL;
}
// Execute any additional module initialization logic
if (hyperion_exec(module) < 0)
{
Py_DECREF(module);
return NULL;
}
return module;
}
// Module deallocation function to clean up per-interpreter state
static void hyperion_free(void* module)
{
// No specific cleanup required in this example
}
static PyModuleDef_Slot hyperion_slots[] = {
{Py_mod_exec, hyperion_exec},
#if (PY_VERSION_HEX >= 0x030C0000)
{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
#endif
{0, NULL}
};
PyObject* EffectModule::PyInit_hyperion()
// Module definition with multi-phase and per-interpreter state
static struct PyModuleDef hyperion_module = {
PyModuleDef_HEAD_INIT,
"hyperion", // Module name
"Hyperion module", // Module docstring
sizeof(hyperion_module_state), // Size of per-interpreter state
EffectModule::effectMethods, // Methods array
NULL, // Slots array (will be added in PyInit_hyperion)
NULL, // Traverse function (optional)
NULL, // Clear function (optional)
hyperion_free // Free function
};
// initialize function for the hyperion module
PyMODINIT_FUNC PyInit_hyperion(void)
{
return PyModule_Create(&moduleDef);
// assign slots to the module definition
hyperion_module.m_slots = hyperion_slots;
// return a new module definition instance
return PyModuleDef_Init(&hyperion_module);
}
void EffectModule::registerHyperionExtensionModule()
@ -49,34 +105,26 @@ PyObject *EffectModule::json2python(const QJsonValue &jsonData)
{
case QJsonValue::Null:
Py_RETURN_NONE;
case QJsonValue::Undefined:
Py_RETURN_NOTIMPLEMENTED;
case QJsonValue::Double:
{
double doubleIntegratlPart;
double doubleFractionalPart = std::modf(jsonData.toDouble(), &doubleIntegratlPart);
if (doubleFractionalPart > std::numeric_limits<double>::epsilon())
double value = jsonData.toDouble();
if (value == static_cast<int>(value)) // If no fractional part, value is equal to its integer representation
{
return Py_BuildValue("d", jsonData.toDouble());
return Py_BuildValue("i", static_cast<int>(value));
}
return Py_BuildValue("i", jsonData.toInt());
return Py_BuildValue("d", value);
}
case QJsonValue::Bool:
return Py_BuildValue("i", jsonData.toBool() ? 1 : 0);
return PyBool_FromLong(jsonData.toBool() ? 1 : 0);
case QJsonValue::String:
return Py_BuildValue("s", jsonData.toString().toUtf8().constData());
case QJsonValue::Object:
{
PyObject * dict= PyDict_New();
QJsonObject objectData = jsonData.toObject();
for (QJsonObject::iterator i = objectData.begin(); i != objectData.end(); ++i)
{
PyObject * obj = json2python(*i);
PyDict_SetItemString(dict, i.key().toStdString().c_str(), obj);
Py_XDECREF(obj);
}
return dict;
}
return PyUnicode_FromString(jsonData.toString().toUtf8().constData());
case QJsonValue::Array:
{
QJsonArray arrayData = jsonData.toArray();
@ -91,6 +139,37 @@ PyObject *EffectModule::json2python(const QJsonValue &jsonData)
}
return list;
}
case QJsonValue::Object: {
// Python's dict
QJsonObject jsonObject = jsonData.toObject();
PyObject* pyDict = PyDict_New();
for (auto it = jsonObject.begin(); it != jsonObject.end(); ++it) {
// Convert key
PyObject* pyKey = PyUnicode_FromString(it.key().toUtf8().constData());
if (!pyKey) {
Py_XDECREF(pyDict);
return nullptr; // Error occurred, return null
}
// Convert value
PyObject* pyValue = json2python(it.value());
if (!pyValue) {
Py_XDECREF(pyKey);
Py_XDECREF(pyDict);
return nullptr; // Error occurred, return null
}
// Add to dictionary
PyDict_SetItem(pyDict, pyKey, pyValue);
Py_XDECREF(pyKey);
Py_XDECREF(pyValue);
}
return pyDict;
}
default:
// Unsupported type
PyErr_SetString(PyExc_TypeError, "Unsupported QJsonValue type.");
return nullptr;
}
assert(false);

View File

@ -44,14 +44,14 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = PyConfig_SetString(&config, &config.program_name, programName);
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
else
#else
Py_SetProgramName(programName);
#endif
{
// set Python module path when exists
// set Python module path when it exists
QString py_path = QDir::cleanPath(qApp->applicationDirPath() + "/../lib/python" + STRINGIFY(PYTHON_VERSION_MAJOR) + "." + STRINGIFY(PYTHON_VERSION_MINOR));
QString py_file = QDir::cleanPath(qApp->applicationDirPath() + "/python" + STRINGIFY(PYTHON_VERSION_MAJOR) + STRINGIFY(PYTHON_VERSION_MINOR) + ".zip");
QString py_framework = QDir::cleanPath(qApp->applicationDirPath() + "/../Frameworks/Python.framework/Versions/Current/lib/python" + STRINGIFY(PYTHON_VERSION_MAJOR) + "." + STRINGIFY(PYTHON_VERSION_MINOR));
@ -68,12 +68,14 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = PyConfig_SetBytesString(&config, &config.home, QSTRING_CSTR(py_file));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
config.module_search_paths_set = 1;
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(py_file.toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
#else
Py_SetPythonHome(Py_DecodeLocale(py_file.toLatin1().data(), nullptr));
@ -85,18 +87,21 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = PyConfig_SetBytesString(&config, &config.home, QSTRING_CSTR(QDir::cleanPath(qApp->applicationDirPath() + "/../")));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
config.module_search_paths_set = 1;
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(QDir(py_path).absolutePath().toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(QDir(py_path + "/lib-dynload").absolutePath().toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
#else
QStringList python_paths;
@ -114,18 +119,21 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = PyConfig_SetBytesString(&config, &config.home, QSTRING_CSTR(QDir::cleanPath(qApp->applicationDirPath() + "/../Frameworks/Python.framework/Versions/Current")));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
config.module_search_paths_set = 1;
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(QDir(py_framework).absolutePath().toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(QDir(py_framework + "/lib-dynload").absolutePath().toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
#else
QStringList python_paths;
@ -146,7 +154,8 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = Py_InitializeFromConfig(&config);
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
PyConfig_Clear(&config);
#endif
@ -154,6 +163,7 @@ PythonInit::PythonInit()
// init Python
Debug(Logger::getInstance("DAEMON"), "Initializing Python interpreter");
Py_InitializeEx(0);
if (!Py_IsInitialized())
{
throw std::runtime_error("Initializing Python failed!");
@ -165,20 +175,25 @@ PythonInit::PythonInit()
#endif
mainThreadState = PyEval_SaveThread();
return;
}
#if (PY_VERSION_HEX >= 0x03080000)
exception:
// Error handling function to replace goto exception
void PythonInit::handlePythonError(PyStatus status, PyConfig& config)
{
Error(Logger::getInstance("DAEMON"), "Initializing Python config failed with error [%s]", status.err_msg);
PyConfig_Clear(&config);
throw std::runtime_error("Initializing Python failed!");
#endif
}
PythonInit::~PythonInit()
{
Debug(Logger::getInstance("DAEMON"), "Cleaning up Python interpreter");
#if (PY_VERSION_HEX < 0x030C0000)
PyEval_RestoreThread(mainThreadState);
Py_Finalize();
#else
PyThreadState_Swap(mainThreadState);
#endif
int rc = Py_FinalizeEx();
}

View File

@ -1,5 +1,6 @@
#include <python/PythonProgram.h>
#include <python/PythonUtils.h>
#include <utils/Logger.h>
#include <QThread>
@ -7,167 +8,177 @@
PyThreadState* mainThreadState;
PythonProgram::PythonProgram(const QString& name, Logger* log) :
_name(name), _log(log), _tstate(nullptr)
_name(name)
, _log(log)
, _tstate(nullptr)
{
// we probably need to wait until mainThreadState is available
while(mainThreadState == nullptr){};
QThread::msleep(10);
while (mainThreadState == nullptr)
{
QThread::msleep(10); // Wait with delay to avoid busy waiting
}
// Create a new subinterpreter for this thread
#if (PY_VERSION_HEX < 0x030C0000)
// get global lock
PyEval_RestoreThread(mainThreadState);
// Initialize a new thread state
_tstate = Py_NewInterpreter();
#else
PyThreadState* prev = PyThreadState_Swap(NULL);
// Create a new interpreter configuration object
PyInterpreterConfig config{};
// Set configuration options
config.use_main_obmalloc = 0;
config.allow_fork = 0;
config.allow_exec = 0;
config.allow_threads = 1;
config.allow_daemon_threads = 0;
config.check_multi_interp_extensions = 1;
config.gil = PyInterpreterConfig_OWN_GIL;
Py_NewInterpreterFromConfig(&_tstate, &config);
#endif
if (_tstate == nullptr)
{
#if (PY_VERSION_HEX >= 0x03020000)
PyThreadState_Swap(mainThreadState);
#if (PY_VERSION_HEX < 0x030C0000)
PyEval_SaveThread();
#else
PyEval_ReleaseLock();
#endif
Error(_log, "Failed to get thread state for %s", QSTRING_CSTR(_name));
return;
}
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(_tstate);
#endif
}
PythonProgram::~PythonProgram()
{
if (!_tstate)
{
return;
// stop sub threads if needed
for (PyThreadState* s = PyInterpreterState_ThreadHead(_tstate->interp), *old = nullptr; s;)
{
if (s == _tstate)
{
s = s->next;
continue;
}
if (old != s)
{
Debug(_log,"ID %s: Waiting on thread %u", QSTRING_CSTR(_name), s->thread_id);
old = s;
}
Py_BEGIN_ALLOW_THREADS;
QThread::msleep(100);
Py_END_ALLOW_THREADS;
s = PyInterpreterState_ThreadHead(_tstate->interp);
}
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState* prev_thread_state = PyThreadState_Swap(_tstate);
#endif
// Clean up the thread state
Py_EndInterpreter(_tstate);
#if (PY_VERSION_HEX >= 0x03020000)
PyThreadState_Swap(mainThreadState);
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(prev_thread_state);
PyEval_SaveThread();
#else
PyEval_ReleaseLock();
#endif
}
void PythonProgram::execute(const QByteArray& python_code)
{
if (!_tstate)
{
return;
}
PyObject *main_module = PyImport_ImportModule("__main__"); // New Reference
PyObject *main_dict = PyModule_GetDict(main_module); // Borrowed reference
Py_INCREF(main_dict); // Incref "main_dict" to use it in PyRun_String(), because PyModule_GetDict() has decref "main_dict"
Py_DECREF(main_module); // // release "main_module" when done
PyObject *result = PyRun_String(python_code.constData(), Py_file_input, main_dict, main_dict); // New Reference
PyThreadState* prev_thread_state = PyThreadState_Swap(_tstate);
PyObject* main_module = PyImport_ImportModule("__main__");
if (!main_module)
{
// Restore the previous thread state
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(mainThreadState);
#else
PyThreadState_Swap(prev_thread_state);
#endif
return;
}
PyObject* main_dict = PyModule_GetDict(main_module); // Borrowed reference to globals
PyObject* result = PyRun_String(python_code.constData(), Py_file_input, main_dict, main_dict);
if (!result)
{
if (PyErr_Occurred()) // Nothing needs to be done for a borrowed reference
if (PyErr_Occurred())
{
Error(_log, "###### PYTHON EXCEPTION ######");
Error(_log, "## In effect '%s'", QSTRING_CSTR(_name));
/* Objects all initialized to NULL for Py_XDECREF */
PyObject* errorType = NULL, * errorValue = NULL, * errorTraceback = NULL;
PyErr_Fetch(&errorType, &errorValue, &errorTraceback); // New Reference or NULL
PyErr_Fetch(&errorType, &errorValue, &errorTraceback);
PyErr_NormalizeException(&errorType, &errorValue, &errorTraceback);
// Extract exception message from "errorValue"
if (errorValue)
{
QString message;
if (PyObject_HasAttrString(errorValue, "__class__"))
{
PyObject *classPtr = PyObject_GetAttrString(errorValue, "__class__"); // New Reference
PyObject *class_name = NULL; /* Object "class_name" initialized to NULL for Py_XDECREF */
class_name = PyObject_GetAttrString(classPtr, "__name__"); // New Reference or NULL
PyObject* classPtr = PyObject_GetAttrString(errorValue, "__class__");
PyObject* class_name = classPtr ? PyObject_GetAttrString(classPtr, "__name__") : NULL;
if (class_name && PyUnicode_Check(class_name))
message.append(PyUnicode_AsUTF8(class_name));
Py_DECREF(classPtr); // release "classPtr" when done
Py_XDECREF(class_name); // Use Py_XDECREF() to ignore NULL references
Py_XDECREF(class_name);
Py_DECREF(classPtr);
}
// Object "class_name" initialized to NULL for Py_XDECREF
PyObject *valueString = NULL;
valueString = PyObject_Str(errorValue); // New Reference or NULL
PyObject* valueString = PyObject_Str(errorValue);
if (valueString && PyUnicode_Check(valueString))
{
if (!message.isEmpty())
message.append(": ");
message.append(PyUnicode_AsUTF8(valueString));
}
Py_XDECREF(valueString); // Use Py_XDECREF() to ignore NULL references
Py_XDECREF(valueString);
Error(_log, "## %s", QSTRING_CSTR(message));
}
// Extract exception message from "errorTraceback"
if (errorTraceback)
{
// Object "tracebackList" initialized to NULL for Py_XDECREF
PyObject *tracebackModule = NULL, *methodName = NULL, *tracebackList = NULL;
QString tracebackMsg;
tracebackModule = PyImport_ImportModule("traceback"); // New Reference or NULL
methodName = PyUnicode_FromString("format_exception"); // New Reference or NULL
tracebackList = PyObject_CallMethodObjArgs(tracebackModule, methodName, errorType, errorValue, errorTraceback, NULL); // New Reference or NULL
PyObject* tracebackModule = PyImport_ImportModule("traceback");
PyObject* methodName = PyUnicode_FromString("format_exception");
PyObject* tracebackList = tracebackModule && methodName
? PyObject_CallMethodObjArgs(tracebackModule, methodName, errorType, errorValue, errorTraceback, NULL)
: NULL;
if (tracebackList)
{
PyObject* iterator = PyObject_GetIter(tracebackList); // New Reference
PyObject* iterator = PyObject_GetIter(tracebackList);
PyObject* item;
while( (item = PyIter_Next(iterator)) ) // New Reference
while ((item = PyIter_Next(iterator)))
{
Error(_log, "## %s", QSTRING_CSTR(QString(PyUnicode_AsUTF8(item)).trimmed()));
Py_DECREF(item); // release "item" when done
Py_DECREF(item);
}
Py_DECREF(iterator); // release "iterator" when done
Py_DECREF(iterator);
}
// Use Py_XDECREF() to ignore NULL references
Py_XDECREF(tracebackModule);
Py_XDECREF(methodName);
Py_XDECREF(tracebackList);
// Give the exception back to python and print it to stderr in case anyone else wants it.
Py_XINCREF(errorType);
Py_XINCREF(errorValue);
Py_XINCREF(errorTraceback);
PyErr_Restore(errorType, errorValue, errorTraceback);
//PyErr_PrintEx(0); // Remove this line to switch off stderr output
}
Error(_log, "###### EXCEPTION END ######");
}
}
else
{
Py_DECREF(result); // release "result" when done
Py_DECREF(result); // Release result when done
}
Py_DECREF(main_dict); // release "main_dict" when done
Py_DECREF(main_module);
// Restore the previous thread state
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(mainThreadState);
#else
PyThreadState_Swap(prev_thread_state);
#endif
}