Python 3.4 (#479)

* Python 3

* fix travis osx

* try fix

* get info

* digging in the dirt

* .

* .

* cleanup

* .

* .

* finalize, add multi threaded python support

.travis/travis_install.sh

@@ -9,6 +9,7 @@ then
 	echo "Install OSX deps"
 	time brew update
 	time brew install qt5 || true
+	time brew install python3 || true
 	time brew install libusb || true
 	time brew install cmake || true
 	time brew install doxygen || true
@@ -18,9 +19,8 @@ elif [[ $TRAVIS_OS_NAME == 'linux' ]]
 then
 	echo "Install linux deps"
 	sudo apt-get -qq update
-	sudo apt-get install -qq -y qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev doxygen expect
+	sudo apt-get install -qq -y qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python3-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev doxygen expect
 else
     echo "Unsupported platform: $TRAVIS_OS_NAME"
     exit 5
 fi
-

CMakeLists.txt

@@ -67,9 +67,24 @@ endif()
 
 message( STATUS "PLATFORM: ${PLATFORM}")
 
+# Macro to get path of first sub dir of a dir, used for MAC OSX lib/header searching
+MACRO(FIRSTSUBDIR result curdir)
+  FILE(GLOB children RELATIVE ${curdir} ${curdir}/*)
+  SET(dirlist "")
+  FOREACH(child ${children})
+    IF(IS_DIRECTORY ${curdir}/${child})
+      LIST(APPEND dirlist "${curdir}/${child}")
+	  #BREAK()
+    ENDIF()
+  ENDFOREACH()
+  SET(${result} ${dirlist})
+ENDMACRO()
 
 if ( "${PLATFORM}" MATCHES "osx" )
-	SET(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/usr/local/opt/qt5" CACHE STRING "path to your QT5 files" )
+	# add specific prefix paths
+	FIRSTSUBDIR(SUBDIRQT "/usr/local/Cellar/qt5")
+	FIRSTSUBDIR(SUBDIRPY "/usr/local/opt/python3/Frameworks/Python.framework/Versions")
+	SET(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${SUBDIRQT} ${SUBDIRPY} "/usr/local/opt/qt5" )
 	include_directories("/opt/X11/include/")
 	SET ( DEFAULT_OSX        ON )
 	SET ( DEFAULT_USB_HID    ON )

CompileHowto.md

@@ -4,7 +4,7 @@
 
 ```
 sudo apt-get update
-sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev
+sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python3-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev
 ```
 ### Ubuntu 14.04 specific
 You need a never version of cmake (minimum 3.0.0). Install it from the ppa or website
@@ -35,6 +35,7 @@ To install on OS X you either need Homebrew or Macport but Homebrew is the recom
 First you need to install the dependencies:
 ```
 brew install qt5
+brew install python3
 brew install cmake
 brew install libusb
 brew install doxygen
@@ -115,11 +116,11 @@ cmake -DENABLE_FB=ON -DCMAKE_BUILD_TYPE=Release ..
 
 To generate make files on OS X:
 
-After which you can run cmake with the correct qt5 path:
+Platform should be auto detected and refer to osx, you can also force osx:
 ```
-export QVER=$(find  /usr/local/Cellar/qt5 -type d -name "5.*" | sort -n  | head -n1)
+cmake -DPLATFORM=osx -DCMAKE_BUILD_TYPE=Release ..
-cmake -DCMAKE_PREFIX_PATH=$QVER  -DCMAKE_BUILD_TYPE=Release ..
 ```
+
 ### Run make to build Hyperion
 The `-j $(nproc)` specifies the amount of CPU cores to use.
 ```bash

README.md

@@ -32,9 +32,14 @@ More information can be found on the official Hyperion [Wiki](https://wiki.hyper
 If you need further support please open a topic at the our new forum!
 [Hyperion webpage/forum](https://www.hyperion-project.org).
 
-## Building
+## Requirements
+* Debian 8, Ubuntu 14.04 or higher. Windows is not supported currently.
 
+## Building
 See [Compilehowto](CompileHowto.md) and [CrossCompileHowto](CrossCompileHowto.txt).
 
+## Download
+A download isn't available, you need to compile your own version see "Building"
+
 ## License
 The source is released under MIT-License (see http://opensource.org/licenses/MIT).

bin/compile.sh

@@ -5,7 +5,7 @@ CFG="${2:-Release}"
 INST="$( [ "${3:-}" = "install" ] && echo true || echo false )"
 
 sudo apt-get update
-sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev  || exit 1
+sudo apt-get install git cmake build-essential qtbase5-dev libqt5serialport5-dev libusb-1.0-0-dev python3-dev libxrender-dev libavahi-core-dev libavahi-compat-libdnssd-dev  || exit 1
 
 if [ -e /dev/vc-cma -a -e /dev/vc-mem ]
 then
@@ -24,4 +24,3 @@ make -j $(nproc)                 || exit 1
 $INST && sudo make install/strip
 echo "to uninstall (not very well tested, please keep that in mind):"
 echo "   sudo make uninstall"
-

cmake/packages.cmake

@@ -15,7 +15,7 @@ SET ( CPACK_DEBIAN_PACKAGE_MAINTAINER "Hyperion Team")
 SET ( CPACK_DEBIAN_PACKAGE_NAME "Hyperion" )
 SET ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/cmake/debian/postinst;${CMAKE_CURRENT_SOURCE_DIR}/cmake/debian/preinst" )
 SET ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://www.hyperion-project.org" )
-SET ( CPACK_DEBIAN_PACKAGE_DEPENDS "libqt5core5a (>= 5.2.0), libqt5network5 (>= 5.2.0), libqt5gui5 (>= 5.2.0), libqt5serialport5 (>= 5.2.0), libavahi-core7 (>= 0.6.31), libavahi-compat-libdnssd1 (>= 0.6.31), libusb-1.0-0, libpython2.7, libc6" )
+SET ( CPACK_DEBIAN_PACKAGE_DEPENDS "libqt5core5a (>= 5.2.0), libqt5network5 (>= 5.2.0), libqt5gui5 (>= 5.2.0), libqt5serialport5 (>= 5.2.0), libavahi-core7 (>= 0.6.31), libavahi-compat-libdnssd1 (>= 0.6.31), libusb-1.0-0, libpython3.4, libc6" )
 SET ( CPACK_DEBIAN_PACKAGE_SECTION "Miscellaneous" )
 
 SET ( CPACK_RPM_PACKAGE_NAME "Hyperion" )

effects/gif.py

@@ -12,5 +12,5 @@ if imageFile:
 # Start the write data loop
 while not hyperion.abort() and imageList:
 	for image in imageList:
-		hyperion.setImage(image.imageWidth, image.imageHeight, image.imageData)
+		hyperion.setImage(image["imageWidth"], image["imageHeight"], image["imageData"])
 		time.sleep(sleepTime)

include/effectengine/EffectEngine.h

@@ -80,7 +80,7 @@ private:
 
 	std::list<EffectSchema> _effectSchemas;
 
-	PyThreadState * _mainThreadState;
-
 	Logger * _log;
+
+	PyThreadState* _mainThreadState;
 };

libsrc/effectengine/CMakeLists.txt

@@ -1,5 +1,4 @@
-find_package(PythonLibs 2.7 REQUIRED)
+find_package(PythonLibs 3.4 REQUIRED)
-message( STATUS "PYTHON VERSIONS FOUND: ${PYTHONLIBS_VERSION_STRING}" )
 
 # Include the python directory. Also include the parent (which is for example /usr/include)
 # which may be required when it is not includes by the (cross-) compiler by default.

libsrc/effectengine/Effect.cpp

@@ -1,6 +1,3 @@
-// Python include
-#include <Python.h>
-
 // stl includes
 #include <iostream>
 #include <sstream>
@@ -51,7 +48,7 @@ PyMethodDef Effect::effectMethods[] = {
 	{NULL, NULL, 0, NULL}
 };
 
-#if PY_MAJOR_VERSION >= 3
+
 // create the hyperion module
 struct PyModuleDef Effect::moduleDef = {
 	PyModuleDef_HEAD_INIT,
@@ -69,20 +66,15 @@ PyObject* Effect::PyInit_hyperion()
 {
 	return PyModule_Create(&moduleDef);
 }
-#else
-void Effect::PyInit_hyperion()
-{
-	Py_InitModule("hyperion", effectMethods);
-}
-#endif
 
 void Effect::registerHyperionExtensionModule()
 {
 	PyImport_AppendInittab("hyperion", &PyInit_hyperion);
 }
 
-Effect::Effect(int priority, int timeout, const QString & script, const QString & name, const QJsonObject & args, const QString & origin, unsigned smoothCfg)
+Effect::Effect(PyThreadState* mainThreadState, int priority, int timeout, const QString & script, const QString & name, const QJsonObject & args, const QString & origin, unsigned smoothCfg)
 	: QThread()
+	, _mainThreadState(mainThreadState)
 	, _priority(priority)
 	, _timeout(timeout)
 	, _script(script)
@@ -90,7 +82,6 @@ Effect::Effect(int priority, int timeout, const QString & script, const QString
 	, _smoothCfg(smoothCfg)
 	, _args(args)
 	, _endTime(-1)
-	, _interpreterThreadState(nullptr)
 	, _imageProcessor(ImageProcessorFactory::getInstance().newImageProcessor())
 	, _colors()
 	, _origin(origin)
@@ -120,11 +111,18 @@ Effect::~Effect()
 
 void Effect::run()
 {
-	// switch to the main thread state and acquire the GIL
+	// get global lock
-	PyEval_AcquireLock();
+	PyEval_RestoreThread(_mainThreadState);
 
 	// Initialize a new thread state
-	_interpreterThreadState = Py_NewInterpreter();
+	PyThreadState* tstate = Py_NewInterpreter();
+	if(tstate == nullptr)
+	{
+		PyEval_ReleaseLock();
+		Error(_log, "Failed to get thread state for %s",QSTRING_CSTR(_name));
+		return;
+	}
+	PyThreadState_Swap(tstate);
 
 	// import the buildtin Hyperion module
 	PyObject * module = PyImport_ImportModule("hyperion");
@@ -193,8 +191,8 @@ void Effect::run()
 						PyObject *class_name = NULL; /* Object "class_name" initialized to NULL for Py_XDECREF */
 						class_name = PyObject_GetAttrString(classPtr, "__name__"); // New Reference or NULL
 
-						if(class_name && PyString_Check(class_name))
+						if(class_name && PyUnicode_Check(class_name))
-							message.append(PyString_AsString(class_name));
+							message.append(PyUnicode_AsUTF8(class_name));
 
 						Py_DECREF(classPtr); // release "classPtr" when done
 						Py_XDECREF(class_name); // Use Py_XDECREF() to ignore NULL references
@@ -204,12 +202,12 @@ void Effect::run()
 					PyObject *valueString = NULL;
 					valueString = PyObject_Str(errorValue); // New Reference or NULL
 
-					if(valueString && PyString_Check(valueString))
+					if(valueString && PyUnicode_Check(valueString))
 					{
 						if(!message.isEmpty())
 							message.append(": ");
 
-						message.append(PyString_AsString(valueString));
+						message.append(PyUnicode_AsUTF8(valueString));
 					}
 					Py_XDECREF(valueString); // Use Py_XDECREF() to ignore NULL references
 
@@ -224,7 +222,7 @@ void Effect::run()
 					QString tracebackMsg;
 
 					tracebackModule = PyImport_ImportModule("traceback"); // New Reference or NULL
-					methodName = PyString_FromString("format_exception"); // New Reference or NULL
+					methodName = PyUnicode_FromString("format_exception"); // New Reference or NULL
 					tracebackList = PyObject_CallMethodObjArgs(tracebackModule, methodName, errorType, errorValue, errorTraceback, NULL); // New Reference or NULL
 
 					if(tracebackList)
@@ -234,7 +232,7 @@ void Effect::run()
 						PyObject* item;
 						while( (item = PyIter_Next(iterator)) ) // New Reference
 						{
-							Error(_log, "## %s",QSTRING_CSTR(QString(PyString_AsString(item)).trimmed()));
+							Error(_log, "## %s",QSTRING_CSTR(QString(PyUnicode_AsUTF8(item)).trimmed()));
 							Py_DECREF(item); // release "item" when done
 						}
 						Py_DECREF(iterator);  // release "iterator" when done
@@ -263,10 +261,29 @@ void Effect::run()
 
 		Py_DECREF(main_dict);  // release "main_dict" when done
 	}
+	// stop sub threads if needed
+	for (PyThreadState* s = tstate->interp->tstate_head, *old = nullptr; s;)
+	{
+		if (s == tstate)
+		{
+			s = s->next;
+			continue;
+		}
+		if (old != s)
+		{
+			Debug(_log,"ID %s: Waiting on thread %u", QSTRING_CSTR(_name), s->thread_id);
+			old = s;
+		}
+
+		Py_BEGIN_ALLOW_THREADS;
+		msleep(100);
+		Py_END_ALLOW_THREADS;
+
+		s = tstate->interp->tstate_head;
+	}
 
 	// Clean up the thread state
-	Py_EndInterpreter(_interpreterThreadState);
+	Py_EndInterpreter(tstate);
-	_interpreterThreadState = nullptr;
 	PyEval_ReleaseLock();
 }
 
@@ -485,12 +502,6 @@ PyObject* Effect::wrapGetImage(PyObject *self, PyObject *args)
 		file = ":/effects/"+file.mid(1);
 
 	QImageReader reader(file);
-	
-	PyObject* prop;
-	static PyObject* imageClass = NULL;
-
-	if (imageClass == NULL)
-		imageClass = PyClass_New(NULL, PyDict_New(), PyString_FromString("ImageClass"));
 
 	if (reader.canRead())
 	{
@@ -502,17 +513,9 @@ PyObject* Effect::wrapGetImage(PyObject *self, PyObject *args)
 			if (reader.canRead())
 			{
 				QImage qimage = reader.read();
-				PyObject* imageDict = PyDict_New();
+
-				
 				int width = qimage.width();
 				int height = qimage.height();
-				
-				prop = Py_BuildValue("i", width);
-				PyDict_SetItemString(imageDict, "imageWidth", prop);
-				Py_XDECREF(prop);
-				prop = Py_BuildValue("i", height);
-				PyDict_SetItemString(imageDict, "imageHeight", prop);
-				Py_XDECREF(prop);
 
 				QByteArray binaryImage;
 				for (int i = 0; i<height; ++i)
@@ -525,12 +528,7 @@ PyObject* Effect::wrapGetImage(PyObject *self, PyObject *args)
 						binaryImage.append((char) qBlue(scanline[j]));
 					}
 				}
-
+				PyList_SET_ITEM(result, i, Py_BuildValue("{s:i,s:i,s:O}", "imageWidth", width, "imageHeight", height, "imageData", PyByteArray_FromStringAndSize(binaryImage.constData(),binaryImage.size())));
-				prop = Py_BuildValue("O", PyByteArray_FromStringAndSize(binaryImage.constData(),binaryImage.size()));
-				PyDict_SetItemString(imageDict, "imageData", prop);
-				Py_XDECREF(prop);
-
-				PyList_SET_ITEM(result, i, Py_BuildValue("O", PyInstance_NewRaw(imageClass, imageDict)));
 			}
 			else
 			{

libsrc/effectengine/Effect.h

@@ -1,7 +1,10 @@
 #pragma once
 
 // Python includes
+// collide of qt slots macro
+#undef slots
 #include <Python.h>
+#define slots
 
 // Qt includes
 #include <QThread>
@@ -19,7 +22,7 @@ class Effect : public QThread
 	Q_OBJECT
 
 public:
-	Effect(int priority, int timeout, const QString & script, const QString & name, const QJsonObject & args = QJsonObject(), const QString & origin="System", unsigned smoothCfg=0);
+	Effect(PyThreadState* mainThreadState, int priority, int timeout, const QString & script, const QString & name, const QJsonObject & args = QJsonObject(), const QString & origin="System", unsigned smoothCfg=0);
 	virtual ~Effect();
 
 	virtual void run();
@@ -70,15 +73,13 @@ private:
 	static PyObject* wrapImageResetT           (PyObject *self, PyObject *args);
 	static Effect * getEffect();
 
-#if PY_MAJOR_VERSION >= 3
 	static struct PyModuleDef moduleDef;
 	static PyObject* PyInit_hyperion();
-#else
-	static void PyInit_hyperion();
-#endif
 
 	void addImage();
 
+	PyThreadState* _mainThreadState;
+
 	const int _priority;
 
 	const int _timeout;
@@ -91,8 +92,6 @@ private:
 
 	int64_t _endTime;
 
-	PyThreadState * _interpreterThreadState;
-
 	/// The processor for translating images to led-values
 	ImageProcessor * _imageProcessor;
 

libsrc/effectengine/EffectEngine.cpp

@@ -24,9 +24,10 @@ EffectEngine::EffectEngine(Hyperion * hyperion, const QJsonObject & jsonEffectCo
 	, _effectConfig(jsonEffectConfig)
 	, _availableEffects()
 	, _activeEffects()
-	, _mainThreadState(nullptr)
 	, _log(Logger::getInstance("EFFECTENGINE"))
+	, _mainThreadState(nullptr)
 {
+
 	Q_INIT_RESOURCE(EffectEngine);
 	qRegisterMetaType<std::vector<ColorRgb>>("std::vector<ColorRgb>");
 	qRegisterMetaType<hyperion::Components>("hyperion::Components");
@@ -280,7 +281,7 @@ int EffectEngine::runEffectScript(const QString &script, const QString &name, co
 	channelCleared(priority);
 
 	// create the effect
-    Effect * effect = new Effect(priority, timeout, script, name, args, origin, smoothCfg);
+    Effect * effect = new Effect(_mainThreadState, priority, timeout, script, name, args, origin, smoothCfg);
 	connect(effect, SIGNAL(setColors(int,std::vector<ColorRgb>,int,bool,hyperion::Components,const QString,unsigned)), _hyperion, SLOT(setColors(int,std::vector<ColorRgb>,int,bool,hyperion::Components,const QString,unsigned)), Qt::QueuedConnection);
 	connect(effect, &QThread::finished, this, &EffectEngine::effectFinished);
 	_activeEffects.push_back(effect);