From 4c2b75b45a8ca5847270db5c96c59875f782f124 Mon Sep 17 00:00:00 2001 From: brindosch Date: Fri, 13 Oct 2017 17:49:29 +0200 Subject: [PATCH] Python 3.4 (#479) * Python 3 * fix travis osx * try fix * get info * digging in the dirt * . * . * cleanup * . * . * finalize, add multi threaded python support --- .travis/travis_install.sh | 4 +- CMakeLists.txt | 17 +++++- CompileHowto.md | 9 +-- README.md | 7 ++- bin/compile.sh | 3 +- cmake/packages.cmake | 2 +- effects/gif.py | 2 +- include/effectengine/EffectEngine.h | 4 +- libsrc/effectengine/CMakeLists.txt | 3 +- libsrc/effectengine/Effect.cpp | 86 ++++++++++++++-------------- libsrc/effectengine/Effect.h | 13 ++--- libsrc/effectengine/EffectEngine.cpp | 5 +- 12 files changed, 86 insertions(+), 69 deletions(-) diff --git a/.travis/travis_install.sh b/.travis/travis_install.sh index 501a03b3..e974ab2e 100755 --- a/.travis/travis_install.sh +++ b/.travis/travis_install.sh @@ -9,6 +9,7 @@ then echo "Install OSX deps" time brew update time brew install qt5 || true + time brew install python3 || true time brew install libusb || true time brew install cmake || true time brew install doxygen || true @@ -18,9 +19,8 @@ elif [[ $TRAVIS_OS_NAME == 'linux' ]] then echo "Install linux deps" sudo apt-get -qq update - sudo apt-get install -qq -y qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev doxygen expect + sudo apt-get install -qq -y qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python3-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev doxygen expect else echo "Unsupported platform: $TRAVIS_OS_NAME" exit 5 fi - diff --git a/CMakeLists.txt b/CMakeLists.txt index 98914fb3..facbd343 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,9 +67,24 @@ endif() message( STATUS "PLATFORM: ${PLATFORM}") +# Macro to get path of first sub dir of a dir, used for MAC OSX lib/header searching +MACRO(FIRSTSUBDIR result curdir) + FILE(GLOB children RELATIVE ${curdir} ${curdir}/*) + SET(dirlist "") + FOREACH(child ${children}) + IF(IS_DIRECTORY ${curdir}/${child}) + LIST(APPEND dirlist "${curdir}/${child}") + #BREAK() + ENDIF() + ENDFOREACH() + SET(${result} ${dirlist}) +ENDMACRO() if ( "${PLATFORM}" MATCHES "osx" ) - SET(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/usr/local/opt/qt5" CACHE STRING "path to your QT5 files" ) + # add specific prefix paths + FIRSTSUBDIR(SUBDIRQT "/usr/local/Cellar/qt5") + FIRSTSUBDIR(SUBDIRPY "/usr/local/opt/python3/Frameworks/Python.framework/Versions") + SET(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${SUBDIRQT} ${SUBDIRPY} "/usr/local/opt/qt5" ) include_directories("/opt/X11/include/") SET ( DEFAULT_OSX ON ) SET ( DEFAULT_USB_HID ON ) diff --git a/CompileHowto.md b/CompileHowto.md index affa4871..20502323 100644 --- a/CompileHowto.md +++ b/CompileHowto.md @@ -4,7 +4,7 @@ ``` sudo apt-get update -sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev +sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python3-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev ``` ### Ubuntu 14.04 specific You need a never version of cmake (minimum 3.0.0). Install it from the ppa or website @@ -35,6 +35,7 @@ To install on OS X you either need Homebrew or Macport but Homebrew is the recom First you need to install the dependencies: ``` brew install qt5 +brew install python3 brew install cmake brew install libusb brew install doxygen @@ -115,11 +116,11 @@ cmake -DENABLE_FB=ON -DCMAKE_BUILD_TYPE=Release .. To generate make files on OS X: -After which you can run cmake with the correct qt5 path: +Platform should be auto detected and refer to osx, you can also force osx: ``` -export QVER=$(find /usr/local/Cellar/qt5 -type d -name "5.*" | sort -n | head -n1) -cmake -DCMAKE_PREFIX_PATH=$QVER -DCMAKE_BUILD_TYPE=Release .. +cmake -DPLATFORM=osx -DCMAKE_BUILD_TYPE=Release .. ``` + ### Run make to build Hyperion The `-j $(nproc)` specifies the amount of CPU cores to use. ```bash diff --git a/README.md b/README.md index cc786d56..63ea3697 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,14 @@ More information can be found on the official Hyperion [Wiki](https://wiki.hyper If you need further support please open a topic at the our new forum! [Hyperion webpage/forum](https://www.hyperion-project.org). -## Building +## Requirements +* Debian 8, Ubuntu 14.04 or higher. Windows is not supported currently. +## Building See [Compilehowto](CompileHowto.md) and [CrossCompileHowto](CrossCompileHowto.txt). +## Download +A download isn't available, you need to compile your own version see "Building" + ## License The source is released under MIT-License (see http://opensource.org/licenses/MIT). diff --git a/bin/compile.sh b/bin/compile.sh index a9b2bde9..1ff70704 100755 --- a/bin/compile.sh +++ b/bin/compile.sh @@ -5,7 +5,7 @@ CFG="${2:-Release}" INST="$( [ "${3:-}" = "install" ] && echo true || echo false )" sudo apt-get update -sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev || exit 1 +sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python3-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev || exit 1 if [ -e /dev/vc-cma -a -e /dev/vc-mem ] then @@ -24,4 +24,3 @@ make -j $(nproc) || exit 1 $INST && sudo make install/strip echo "to uninstall (not very well tested, please keep that in mind):" echo " sudo make uninstall" - diff --git a/cmake/packages.cmake b/cmake/packages.cmake index ed1d6976..7fef5d09 100644 --- a/cmake/packages.cmake +++ b/cmake/packages.cmake @@ -15,7 +15,7 @@ SET ( CPACK_DEBIAN_PACKAGE_MAINTAINER "Hyperion Team") SET ( CPACK_DEBIAN_PACKAGE_NAME "Hyperion" ) SET ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/cmake/debian/postinst;${CMAKE_CURRENT_SOURCE_DIR}/cmake/debian/preinst" ) SET ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://www.hyperion-project.org" ) -SET ( CPACK_DEBIAN_PACKAGE_DEPENDS "libqt5core5a (>= 5.2.0), libqt5network5 (>= 5.2.0), libqt5gui5 (>= 5.2.0), libqt5serialport5 (>= 5.2.0), libavahi-core7 (>= 0.6.31), libavahi-compat-libdnssd1 (>= 0.6.31), libusb-1.0-0, libpython2.7, libc6" ) +SET ( CPACK_DEBIAN_PACKAGE_DEPENDS "libqt5core5a (>= 5.2.0), libqt5network5 (>= 5.2.0), libqt5gui5 (>= 5.2.0), libqt5serialport5 (>= 5.2.0), libavahi-core7 (>= 0.6.31), libavahi-compat-libdnssd1 (>= 0.6.31), libusb-1.0-0, libpython3.4, libc6" ) SET ( CPACK_DEBIAN_PACKAGE_SECTION "Miscellaneous" ) SET ( CPACK_RPM_PACKAGE_NAME "Hyperion" ) diff --git a/effects/gif.py b/effects/gif.py index e89621a0..4463f782 100644 --- a/effects/gif.py +++ b/effects/gif.py @@ -12,5 +12,5 @@ if imageFile: # Start the write data loop while not hyperion.abort() and imageList: for image in imageList: - hyperion.setImage(image.imageWidth, image.imageHeight, image.imageData) + hyperion.setImage(image["imageWidth"], image["imageHeight"], image["imageData"]) time.sleep(sleepTime) diff --git a/include/effectengine/EffectEngine.h b/include/effectengine/EffectEngine.h index 2f5494a7..323b8270 100644 --- a/include/effectengine/EffectEngine.h +++ b/include/effectengine/EffectEngine.h @@ -80,7 +80,7 @@ private: std::list _effectSchemas; - PyThreadState * _mainThreadState; - Logger * _log; + + PyThreadState* _mainThreadState; }; diff --git a/libsrc/effectengine/CMakeLists.txt b/libsrc/effectengine/CMakeLists.txt index 9a6b1652..c016b4cc 100644 --- a/libsrc/effectengine/CMakeLists.txt +++ b/libsrc/effectengine/CMakeLists.txt @@ -1,5 +1,4 @@ -find_package(PythonLibs 2.7 REQUIRED) -message( STATUS "PYTHON VERSIONS FOUND: ${PYTHONLIBS_VERSION_STRING}" ) +find_package(PythonLibs 3.4 REQUIRED) # Include the python directory. Also include the parent (which is for example /usr/include) # which may be required when it is not includes by the (cross-) compiler by default. diff --git a/libsrc/effectengine/Effect.cpp b/libsrc/effectengine/Effect.cpp index c15a0c29..fda2f934 100644 --- a/libsrc/effectengine/Effect.cpp +++ b/libsrc/effectengine/Effect.cpp @@ -1,6 +1,3 @@ -// Python include -#include - // stl includes #include #include @@ -51,7 +48,7 @@ PyMethodDef Effect::effectMethods[] = { {NULL, NULL, 0, NULL} }; -#if PY_MAJOR_VERSION >= 3 + // create the hyperion module struct PyModuleDef Effect::moduleDef = { PyModuleDef_HEAD_INIT, @@ -69,20 +66,15 @@ PyObject* Effect::PyInit_hyperion() { return PyModule_Create(&moduleDef); } -#else -void Effect::PyInit_hyperion() -{ - Py_InitModule("hyperion", effectMethods); -} -#endif void Effect::registerHyperionExtensionModule() { PyImport_AppendInittab("hyperion", &PyInit_hyperion); } -Effect::Effect(int priority, int timeout, const QString & script, const QString & name, const QJsonObject & args, const QString & origin, unsigned smoothCfg) +Effect::Effect(PyThreadState* mainThreadState, int priority, int timeout, const QString & script, const QString & name, const QJsonObject & args, const QString & origin, unsigned smoothCfg) : QThread() + , _mainThreadState(mainThreadState) , _priority(priority) , _timeout(timeout) , _script(script) @@ -90,7 +82,6 @@ Effect::Effect(int priority, int timeout, const QString & script, const QString , _smoothCfg(smoothCfg) , _args(args) , _endTime(-1) - , _interpreterThreadState(nullptr) , _imageProcessor(ImageProcessorFactory::getInstance().newImageProcessor()) , _colors() , _origin(origin) @@ -120,11 +111,18 @@ Effect::~Effect() void Effect::run() { - // switch to the main thread state and acquire the GIL - PyEval_AcquireLock(); + // get global lock + PyEval_RestoreThread(_mainThreadState); // Initialize a new thread state - _interpreterThreadState = Py_NewInterpreter(); + PyThreadState* tstate = Py_NewInterpreter(); + if(tstate == nullptr) + { + PyEval_ReleaseLock(); + Error(_log, "Failed to get thread state for %s",QSTRING_CSTR(_name)); + return; + } + PyThreadState_Swap(tstate); // import the buildtin Hyperion module PyObject * module = PyImport_ImportModule("hyperion"); @@ -193,8 +191,8 @@ void Effect::run() PyObject *class_name = NULL; /* Object "class_name" initialized to NULL for Py_XDECREF */ class_name = PyObject_GetAttrString(classPtr, "__name__"); // New Reference or NULL - if(class_name && PyString_Check(class_name)) - message.append(PyString_AsString(class_name)); + if(class_name && PyUnicode_Check(class_name)) + message.append(PyUnicode_AsUTF8(class_name)); Py_DECREF(classPtr); // release "classPtr" when done Py_XDECREF(class_name); // Use Py_XDECREF() to ignore NULL references @@ -204,12 +202,12 @@ void Effect::run() PyObject *valueString = NULL; valueString = PyObject_Str(errorValue); // New Reference or NULL - if(valueString && PyString_Check(valueString)) + if(valueString && PyUnicode_Check(valueString)) { if(!message.isEmpty()) message.append(": "); - message.append(PyString_AsString(valueString)); + message.append(PyUnicode_AsUTF8(valueString)); } Py_XDECREF(valueString); // Use Py_XDECREF() to ignore NULL references @@ -224,7 +222,7 @@ void Effect::run() QString tracebackMsg; tracebackModule = PyImport_ImportModule("traceback"); // New Reference or NULL - methodName = PyString_FromString("format_exception"); // New Reference or NULL + methodName = PyUnicode_FromString("format_exception"); // New Reference or NULL tracebackList = PyObject_CallMethodObjArgs(tracebackModule, methodName, errorType, errorValue, errorTraceback, NULL); // New Reference or NULL if(tracebackList) @@ -234,7 +232,7 @@ void Effect::run() PyObject* item; while( (item = PyIter_Next(iterator)) ) // New Reference { - Error(_log, "## %s",QSTRING_CSTR(QString(PyString_AsString(item)).trimmed())); + Error(_log, "## %s",QSTRING_CSTR(QString(PyUnicode_AsUTF8(item)).trimmed())); Py_DECREF(item); // release "item" when done } Py_DECREF(iterator); // release "iterator" when done @@ -263,10 +261,29 @@ void Effect::run() Py_DECREF(main_dict); // release "main_dict" when done } + // stop sub threads if needed + for (PyThreadState* s = tstate->interp->tstate_head, *old = nullptr; s;) + { + if (s == tstate) + { + s = s->next; + continue; + } + if (old != s) + { + Debug(_log,"ID %s: Waiting on thread %u", QSTRING_CSTR(_name), s->thread_id); + old = s; + } + + Py_BEGIN_ALLOW_THREADS; + msleep(100); + Py_END_ALLOW_THREADS; + + s = tstate->interp->tstate_head; + } // Clean up the thread state - Py_EndInterpreter(_interpreterThreadState); - _interpreterThreadState = nullptr; + Py_EndInterpreter(tstate); PyEval_ReleaseLock(); } @@ -485,12 +502,6 @@ PyObject* Effect::wrapGetImage(PyObject *self, PyObject *args) file = ":/effects/"+file.mid(1); QImageReader reader(file); - - PyObject* prop; - static PyObject* imageClass = NULL; - - if (imageClass == NULL) - imageClass = PyClass_New(NULL, PyDict_New(), PyString_FromString("ImageClass")); if (reader.canRead()) { @@ -502,17 +513,9 @@ PyObject* Effect::wrapGetImage(PyObject *self, PyObject *args) if (reader.canRead()) { QImage qimage = reader.read(); - PyObject* imageDict = PyDict_New(); - + int width = qimage.width(); int height = qimage.height(); - - prop = Py_BuildValue("i", width); - PyDict_SetItemString(imageDict, "imageWidth", prop); - Py_XDECREF(prop); - prop = Py_BuildValue("i", height); - PyDict_SetItemString(imageDict, "imageHeight", prop); - Py_XDECREF(prop); QByteArray binaryImage; for (int i = 0; i +#define slots // Qt includes #include @@ -19,7 +22,7 @@ class Effect : public QThread Q_OBJECT public: - Effect(int priority, int timeout, const QString & script, const QString & name, const QJsonObject & args = QJsonObject(), const QString & origin="System", unsigned smoothCfg=0); + Effect(PyThreadState* mainThreadState, int priority, int timeout, const QString & script, const QString & name, const QJsonObject & args = QJsonObject(), const QString & origin="System", unsigned smoothCfg=0); virtual ~Effect(); virtual void run(); @@ -70,15 +73,13 @@ private: static PyObject* wrapImageResetT (PyObject *self, PyObject *args); static Effect * getEffect(); -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduleDef; static PyObject* PyInit_hyperion(); -#else - static void PyInit_hyperion(); -#endif void addImage(); + PyThreadState* _mainThreadState; + const int _priority; const int _timeout; @@ -91,8 +92,6 @@ private: int64_t _endTime; - PyThreadState * _interpreterThreadState; - /// The processor for translating images to led-values ImageProcessor * _imageProcessor; diff --git a/libsrc/effectengine/EffectEngine.cpp b/libsrc/effectengine/EffectEngine.cpp index d5dac8ed..bfa35c31 100644 --- a/libsrc/effectengine/EffectEngine.cpp +++ b/libsrc/effectengine/EffectEngine.cpp @@ -24,9 +24,10 @@ EffectEngine::EffectEngine(Hyperion * hyperion, const QJsonObject & jsonEffectCo , _effectConfig(jsonEffectConfig) , _availableEffects() , _activeEffects() - , _mainThreadState(nullptr) , _log(Logger::getInstance("EFFECTENGINE")) + , _mainThreadState(nullptr) { + Q_INIT_RESOURCE(EffectEngine); qRegisterMetaType>("std::vector"); qRegisterMetaType("hyperion::Components"); @@ -280,7 +281,7 @@ int EffectEngine::runEffectScript(const QString &script, const QString &name, co channelCleared(priority); // create the effect - Effect * effect = new Effect(priority, timeout, script, name, args, origin, smoothCfg); + Effect * effect = new Effect(_mainThreadState, priority, timeout, script, name, args, origin, smoothCfg); connect(effect, SIGNAL(setColors(int,std::vector,int,bool,hyperion::Components,const QString,unsigned)), _hyperion, SLOT(setColors(int,std::vector,int,bool,hyperion::Components,const QString,unsigned)), Qt::QueuedConnection); connect(effect, &QThread::finished, this, &EffectEngine::effectFinished); _activeEffects.push_back(effect);