frodovdr/hyperion.ng
1
0
Fork 0
mirror of https://github.com/hyperion-project/hyperion.ng.git synced 2025-03-01 10:33:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor Python for 3.12 integration (#1807)

Browse Source 
* Correct JS requestConfig call

* Update requestWriteConfig to new API format

* Add hyperion-light and bare-minimum preset scenarios

* Refactor Python

* Windows add bcrypt until mbedtls  is fixed
(https://github.com/Mbed-TLS/mbedtls/pull/9554)

* Corrections

* Use ScreenCaptureKit under macOS 15 and above

* ReSigning macOS package

* Python 3.11.10 test

* Revert "Python 3.11.10 test"

This reverts commit ee921e4f12.

* Handle defined exits from python scripts

* Update change.log

* CodeQL findings

---------

Co-authored-by: Paulchen-Panther <16664240+Paulchen-Panther@users.noreply.github.com>
This commit is contained in:
LordGrey
2024-12-01 17:08:25 +01:00
committed by GitHub
parent 6e3357ea2d
commit 733aa662bf
18 changed files with 841 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
libsrc/python/PythonInit.cpp
View File

@@ -18,7 +18,7 @@
#include <HyperionConfig.h>
#ifdef _WIN32
#include <stdexcept>
#include <stdexcept>
#endif
#define STRINGIFY2(x) #x
@@ -44,14 +44,14 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = PyConfig_SetString(&config, &config.program_name, programName);
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
else
#else
Py_SetProgramName(programName);
#endif
{
// set Python module path when exists
// set Python module path when it exists
QString py_path = QDir::cleanPath(qApp->applicationDirPath() + "/../lib/python" + STRINGIFY(PYTHON_VERSION_MAJOR) + "." + STRINGIFY(PYTHON_VERSION_MINOR));
QString py_file = QDir::cleanPath(qApp->applicationDirPath() + "/python" + STRINGIFY(PYTHON_VERSION_MAJOR) + STRINGIFY(PYTHON_VERSION_MINOR) + ".zip");
QString py_framework = QDir::cleanPath(qApp->applicationDirPath() + "/../Frameworks/Python.framework/Versions/Current/lib/python" + STRINGIFY(PYTHON_VERSION_MAJOR) + "." + STRINGIFY(PYTHON_VERSION_MINOR));
@@ -59,21 +59,23 @@ PythonInit::PythonInit()
if (QFile(py_file).exists() || QDir(py_path).exists() || QDir(py_framework).exists())
{
#if (PY_VERSION_HEX >= 0x030C0000)
config.site_import = 0;
config.site_import = 0;
#else
Py_NoSiteFlag++;
Py_NoSiteFlag++;
#endif
if (QFile(py_file).exists()) // Windows
{
#if (PY_VERSION_HEX >= 0x03080000)
status = PyConfig_SetBytesString(&config, &config.home, QSTRING_CSTR(py_file));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
config.module_search_paths_set = 1;
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(py_file.toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
#else
Py_SetPythonHome(Py_DecodeLocale(py_file.toLatin1().data(), nullptr));
@@ -85,18 +87,21 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = PyConfig_SetBytesString(&config, &config.home, QSTRING_CSTR(QDir::cleanPath(qApp->applicationDirPath() + "/../")));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
config.module_search_paths_set = 1;
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(QDir(py_path).absolutePath().toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(QDir(py_path + "/lib-dynload").absolutePath().toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
#else
QStringList python_paths;
@@ -114,18 +119,21 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = PyConfig_SetBytesString(&config, &config.home, QSTRING_CSTR(QDir::cleanPath(qApp->applicationDirPath() + "/../Frameworks/Python.framework/Versions/Current")));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
config.module_search_paths_set = 1;
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(QDir(py_framework).absolutePath().toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
status = PyWideStringList_Append(&config.module_search_paths, const_cast<wchar_t*>(QDir(py_framework + "/lib-dynload").absolutePath().toStdWString().c_str()));
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
#else
QStringList python_paths;
@@ -146,7 +154,8 @@ PythonInit::PythonInit()
#if (PY_VERSION_HEX >= 0x03080000)
status = Py_InitializeFromConfig(&config);
if (PyStatus_Exception(status)) {
goto exception;
handlePythonError(status, config);
return;
}
PyConfig_Clear(&config);
#endif
@@ -154,7 +163,8 @@ PythonInit::PythonInit()
// init Python
Debug(Logger::getInstance("DAEMON"), "Initializing Python interpreter");
Py_InitializeEx(0);
if ( !Py_IsInitialized() )
if (!Py_IsInitialized())
{
throw std::runtime_error("Initializing Python failed!");
}
@@ -165,20 +175,28 @@ PythonInit::PythonInit()
#endif
mainThreadState = PyEval_SaveThread();
return;
}
// Error handling function to replace goto exception
#if (PY_VERSION_HEX >= 0x03080000)
exception:
void PythonInit::handlePythonError(PyStatus status, PyConfig& config)
{
Error(Logger::getInstance("DAEMON"), "Initializing Python config failed with error [%s]", status.err_msg);
PyConfig_Clear(&config);
throw std::runtime_error("Initializing Python failed!");
#endif
}
#endif
PythonInit::~PythonInit()
{
Debug(Logger::getInstance("DAEMON"), "Cleaning up Python interpreter");
#if (PY_VERSION_HEX < 0x030C0000)
PyEval_RestoreThread(mainThreadState);
Py_Finalize();
#else
PyThreadState_Swap(mainThreadState);
#endif
int rc = Py_FinalizeEx();
Debug(Logger::getInstance("DAEMON"), "Cleaning up Python interpreter %s", rc == 0 ? "succeeded" : "failed");
}

253
libsrc/python/PythonProgram.cpp
View File

@@ -1,173 +1,238 @@
#include <python/PythonProgram.h>
#include <python/PythonUtils.h>
#include <utils/Logger.h>
#include <QThread>
PyThreadState* mainThreadState;
PythonProgram::PythonProgram(const QString & name, Logger * log) :
_name(name), _log(log), _tstate(nullptr)
PythonProgram::PythonProgram(const QString& name, Logger* log) :
_name(name)
, _log(log)
, _tstate(nullptr)
{
// we probably need to wait until mainThreadState is available
while(mainThreadState == nullptr){};
QThread::msleep(10);
while (mainThreadState == nullptr)
{
QThread::msleep(10); // Wait with delay to avoid busy waiting
}
// Create a new subinterpreter for this thread
#if (PY_VERSION_HEX < 0x030C0000)
// get global lock
PyEval_RestoreThread(mainThreadState);
// Initialize a new thread state
_tstate = Py_NewInterpreter();
if(_tstate == nullptr)
{
#if (PY_VERSION_HEX >= 0x03020000)
PyThreadState_Swap(mainThreadState);
PyEval_SaveThread();
#else
PyEval_ReleaseLock();
PyThreadState* prev = PyThreadState_Swap(NULL);
// Create a new interpreter configuration object
PyInterpreterConfig config{};
// Set configuration options
config.use_main_obmalloc = 0;
config.allow_fork = 0;
config.allow_exec = 0;
config.allow_threads = 1;
config.allow_daemon_threads = 0;
config.check_multi_interp_extensions = 1;
config.gil = PyInterpreterConfig_OWN_GIL;
Py_NewInterpreterFromConfig(&_tstate, &config);
#endif
Error(_log, "Failed to get thread state for %s",QSTRING_CSTR(_name));
if (_tstate == nullptr)
{
PyThreadState_Swap(mainThreadState);
#if (PY_VERSION_HEX < 0x030C0000)
PyEval_SaveThread();
#endif
Error(_log, "Failed to get thread state for %s", QSTRING_CSTR(_name));
return;
}
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(_tstate);
#endif
}
PythonProgram::~PythonProgram()
{
if (!_tstate)
return;
// stop sub threads if needed
for (PyThreadState* s = PyInterpreterState_ThreadHead(_tstate->interp), *old = nullptr; s;)
{
if (s == _tstate)
{
s = s->next;
continue;
}
if (old != s)
{
Debug(_log,"ID %s: Waiting on thread %u", QSTRING_CSTR(_name), s->thread_id);
old = s;
}
Py_BEGIN_ALLOW_THREADS;
QThread::msleep(100);
Py_END_ALLOW_THREADS;
s = PyInterpreterState_ThreadHead(_tstate->interp);
return;
}
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState* prev_thread_state = PyThreadState_Swap(_tstate);
#endif
// Clean up the thread state
Py_EndInterpreter(_tstate);
#if (PY_VERSION_HEX >= 0x03020000)
PyThreadState_Swap(mainThreadState);
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(prev_thread_state);
PyEval_SaveThread();
#else
PyEval_ReleaseLock();
#endif
}
void PythonProgram::execute(const QByteArray & python_code)
void PythonProgram::execute(const QByteArray& python_code)
{
if (!_tstate)
{
return;
}
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(_tstate);
#else
PyThreadState* prev_thread_state = PyThreadState_Swap(_tstate);
#endif
PyObject* main_module = PyImport_ImportModule("__main__");
if (!main_module)
{
// Restore the previous thread state
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(mainThreadState);
#else
PyThreadState_Swap(prev_thread_state);
#endif
return;
}
PyObject *main_module = PyImport_ImportModule("__main__"); // New Reference
PyObject *main_dict = PyModule_GetDict(main_module); // Borrowed reference
Py_INCREF(main_dict); // Incref "main_dict" to use it in PyRun_String(), because PyModule_GetDict() has decref "main_dict"
Py_DECREF(main_module); // // release "main_module" when done
PyObject *result = PyRun_String(python_code.constData(), Py_file_input, main_dict, main_dict); // New Reference
PyObject* main_dict = PyModule_GetDict(main_module); // Borrowed reference to globals
PyObject* result = PyRun_String(python_code.constData(), Py_file_input, main_dict, main_dict);
if (!result)
{
if (PyErr_Occurred()) // Nothing needs to be done for a borrowed reference
if (PyErr_Occurred())
{
Error(_log,"###### PYTHON EXCEPTION ######");
Error(_log,"## In effect '%s'", QSTRING_CSTR(_name));
/* Objects all initialized to NULL for Py_XDECREF */
PyObject *errorType = NULL, *errorValue = NULL, *errorTraceback = NULL;
PyObject* errorType = NULL, * errorValue = NULL, * errorTraceback = NULL;
PyErr_Fetch(&errorType, &errorValue, &errorTraceback); // New Reference or NULL
PyErr_Fetch(&errorType, &errorValue, &errorTraceback);
PyErr_NormalizeException(&errorType, &errorValue, &errorTraceback);
// Extract exception message from "errorValue"
if(errorValue)
{
QString message;
if(PyObject_HasAttrString(errorValue, "__class__"))
{
PyObject *classPtr = PyObject_GetAttrString(errorValue, "__class__"); // New Reference
PyObject *class_name = NULL; /* Object "class_name" initialized to NULL for Py_XDECREF */
class_name = PyObject_GetAttrString(classPtr, "__name__"); // New Reference or NULL
// Check if the exception is a SystemExit
PyObject* systemExitType = PyExc_SystemExit;
bool isSystemExit = PyObject_IsInstance(errorValue, systemExitType);
if(class_name && PyUnicode_Check(class_name))
if (isSystemExit)
{
// Extract the exit argument
PyObject* exitArg = PyObject_GetAttrString(errorValue, "code");
if (exitArg)
{
QString logErrorText;
if (PyTuple_Check(exitArg)) {
PyObject* errorMessage = PyTuple_GetItem(exitArg, 0); // Borrowed reference
PyObject* exitCode = PyTuple_GetItem(exitArg, 1); // Borrowed reference
if (exitCode && PyLong_Check(exitCode))
{
logErrorText = QString("[%1]: ").arg(PyLong_AsLong(exitCode));
}
if (errorMessage && PyUnicode_Check(errorMessage)) {
logErrorText.append(PyUnicode_AsUTF8(errorMessage));
}
}
else if (PyUnicode_Check(exitArg)) {
// If the code is just a string, treat it as an error message
logErrorText.append(PyUnicode_AsUTF8(exitArg));
}
else if (PyLong_Check(exitArg)) {
// If the code is just an integer, treat it as an exit code
logErrorText = QString("[%1]").arg(PyLong_AsLong(exitArg));
}
Error(_log, "Effect '%s' failed with error %s", QSTRING_CSTR(_name), QSTRING_CSTR(logErrorText));
Py_DECREF(exitArg); // Release the reference
}
else
{
Debug(_log, "No 'code' attribute found on SystemExit exception.");
}
// Clear the error so it won't propagate
PyErr_Clear();
Py_DECREF(systemExitType);
return;
}
Py_DECREF(systemExitType);
if (errorValue)
{
Error(_log, "###### PYTHON EXCEPTION ######");
Error(_log, "## In effect '%s'", QSTRING_CSTR(_name));
QString message;
if (PyObject_HasAttrString(errorValue, "__class__"))
{
PyObject* classPtr = PyObject_GetAttrString(errorValue, "__class__");
PyObject* class_name = classPtr ? PyObject_GetAttrString(classPtr, "__name__") : NULL;
if (class_name && PyUnicode_Check(class_name))
message.append(PyUnicode_AsUTF8(class_name));
Py_DECREF(classPtr); // release "classPtr" when done
Py_XDECREF(class_name); // Use Py_XDECREF() to ignore NULL references
Py_XDECREF(class_name);
Py_DECREF(classPtr);
}
// Object "class_name" initialized to NULL for Py_XDECREF
PyObject *valueString = NULL;
valueString = PyObject_Str(errorValue); // New Reference or NULL
PyObject* valueString = PyObject_Str(errorValue);
if(valueString && PyUnicode_Check(valueString))
if (valueString && PyUnicode_Check(valueString))
{
if(!message.isEmpty())
if (!message.isEmpty())
message.append(": ");
message.append(PyUnicode_AsUTF8(valueString));
}
Py_XDECREF(valueString); // Use Py_XDECREF() to ignore NULL references
Py_XDECREF(valueString);
Error(_log, "## %s", QSTRING_CSTR(message));
}
// Extract exception message from "errorTraceback"
if(errorTraceback)
if (errorTraceback)
{
// Object "tracebackList" initialized to NULL for Py_XDECREF
PyObject *tracebackModule = NULL, *methodName = NULL, *tracebackList = NULL;
QString tracebackMsg;
PyObject* tracebackModule = PyImport_ImportModule("traceback");
PyObject* methodName = PyUnicode_FromString("format_exception");
PyObject* tracebackList = tracebackModule && methodName
? PyObject_CallMethodObjArgs(tracebackModule, methodName, errorType, errorValue, errorTraceback, NULL)
: NULL;
tracebackModule = PyImport_ImportModule("traceback"); // New Reference or NULL
methodName = PyUnicode_FromString("format_exception"); // New Reference or NULL
tracebackList = PyObject_CallMethodObjArgs(tracebackModule, methodName, errorType, errorValue, errorTraceback, NULL); // New Reference or NULL
if(tracebackList)
if (tracebackList)
{
PyObject* iterator = PyObject_GetIter(tracebackList); // New Reference
PyObject* iterator = PyObject_GetIter(tracebackList);
PyObject* item;
while( (item = PyIter_Next(iterator)) ) // New Reference
while ((item = PyIter_Next(iterator)))
{
Error(_log, "## %s",QSTRING_CSTR(QString(PyUnicode_AsUTF8(item)).trimmed()));
Py_DECREF(item); // release "item" when done
Error(_log, "## %s", QSTRING_CSTR(QString(PyUnicode_AsUTF8(item)).trimmed()));
Py_DECREF(item);
}
Py_DECREF(iterator); // release "iterator" when done
Py_DECREF(iterator);
}
// Use Py_XDECREF() to ignore NULL references
Py_XDECREF(tracebackModule);
Py_XDECREF(methodName);
Py_XDECREF(tracebackList);
// Give the exception back to python and print it to stderr in case anyone else wants it.
Py_XINCREF(errorType);
Py_XINCREF(errorValue);
Py_XINCREF(errorTraceback);
PyErr_Restore(errorType, errorValue, errorTraceback);
//PyErr_PrintEx(0); // Remove this line to switch off stderr output
}
Error(_log,"###### EXCEPTION END ######");
Error(_log, "###### EXCEPTION END ######");
}
// Clear the error so it won't propagate
PyErr_Clear();
}
else
{
Py_DECREF(result); // release "result" when done
Py_DECREF(result); // Release result when done
}
Py_DECREF(main_dict); // release "main_dict" when done
Py_DECREF(main_module);
// Restore the previous thread state
#if (PY_VERSION_HEX < 0x030C0000)
PyThreadState_Swap(mainThreadState);
#else
PyThreadState_Swap(prev_thread_state);
#endif
}