JSON Auto correction + hyperion schema split for better readability (#452)

* revoke schema split * add "getAutoCorrectedConfig" function * revoke schema split * revoke schema split * revoke schema split * Prevent compiler error if none grabber is available * revoke schema split * add "getAutoCorrectedConfig" function * revoke schema split * remove "configMigrator" * remove "configMigrator" * Change TestConfigFile to show how the autocorrection works * revoke schema split * revoke schema split * remove "ConfigMigrator" * remove "ConfigMigrator" * remove "ConfigMigratorBase" * remove "ConfigMigratorBase" * Add QJsonUtils.h * added ability "ignore-required" It has been added the ability to correct the configuration without having to pay attention to the keyword "required" in the hyperion schema * Allow Comments in Hyperion Schema * add ability to ignore the "required" keyword in hyperion schema * add ability to ignore the "required" keyword in hyperion schema * add ability to ignore the "required" keyword in hyperion schema * //Allow Comments in Hyperion Schema * Update jsonschema.py to version 0.8.0 to support ... references in json schema * add RefResolver from jsonschema.py to resolve references in hyperion schema * remove dupe code * split the hyperion schema in separatly files For better readability * add function "resolveReferences" to resolve references in hyperion schema. * remove CURRENT_CONFIG_VERSION * remove CURRENT_CONFIG_VERSION * split the hyperion schema in separatly files For better readability * Create schema-backgroundEffect.json * Add the rest of the Hyperion schema via upload * Remove Comments in config file * Add return variable to function writeJson(). fix function resolveReferences(). edit function load() to handle QPair result from schemaChecker. * edit function validate() to return QPair variable * fit function loadEffectDefinition() * fit function checkJson() * Expand error check by dividing "_error" variable in "_error" and "_schemaError". Replace variable "bool" in validate() in QPair * Extend function "cmd_cfg_set" to handle auto correction * Extend function "loadConfig" to handle auto correction * fix function loadConfig()
2025-03-01 10:33:28 +00:00 · 2017-07-30 13:32:10 +02:00
parent 622a171808
commit 5bd020a570
45 changed files with 2356 additions and 2254 deletions
--- a/libsrc/utils/CMakeLists.txt
+++ b/libsrc/utils/CMakeLists.txt
@@ -27,6 +27,7 @@ SET(Utils_HEADERS
 	${CURRENT_HEADER_DIR}/RgbToRgbw.h
 	${CURRENT_HEADER_DIR}/jsonschema/QJsonFactory.h
 	${CURRENT_HEADER_DIR}/jsonschema/QJsonSchemaChecker.h
+	${CURRENT_HEADER_DIR}/jsonschema/QJsonUtils.h
 	${CURRENT_HEADER_DIR}/global_defines.h
 	${CURRENT_HEADER_DIR}/SysInfo.h
 )
--- a/libsrc/utils/JsonProcessor.cpp
+++ b/libsrc/utils/JsonProcessor.cpp
@@ -15,7 +15,6 @@
 #include <QFile>
 #include <QFileInfo>
 #include <QJsonDocument>
-#include <QVariantMap>
 #include <QDir>
 #include <QImage>
 #include <QBuffer>
@@ -594,6 +593,7 @@ void JsonProcessor::handleServerInfoCommand(const QJsonObject&, const QString& c
 	ledDevices["available"] = availableLedDevices;
 	info["ledDevices"] = ledDevices;

+#if defined(ENABLE_DISPMANX) || defined(ENABLE_V4L2) || defined(ENABLE_FB) || defined(ENABLE_AMLOGIC) || defined(ENABLE_OSX) || defined(ENABLE_X11)
 	// get available grabbers
 	QJsonObject grabbers;
 	//grabbers["active"] = ????;
@@ -605,6 +605,9 @@ void JsonProcessor::handleServerInfoCommand(const QJsonObject&, const QString& c

 	grabbers["available"] = availableGrabbers;
 	info["grabbers"] = grabbers;
+#else
+	info["grabbers"] = QString("none");
+#endif

 	// get available components
 	QJsonArray component;
@@ -844,45 +847,19 @@ void JsonProcessor::handleSchemaGetCommand(const QJsonObject& message, const QSt

 	// make sure the resources are loaded (they may be left out after static linking)
 	Q_INIT_RESOURCE(resource);
-	QJsonParseError error;

 	// read the hyperion json schema from the resource
-	QFile schemaData(":/hyperion-schema-"+QString::number(_hyperion->getConfigVersionId()));
+	QString schemaFile = ":/hyperion-schema";

-	if (!schemaData.open(QIODevice::ReadOnly))
+	try
 	{
-		std::stringstream error;
-		error << "Schema not found: " << schemaData.errorString().toStdString();
-		throw std::runtime_error(error.str());
+		schemaJson = QJsonFactory::readSchema(schemaFile);
 	}
-
-	QByteArray schema = schemaData.readAll();
-	QJsonDocument doc = QJsonDocument::fromJson(schema, &error);
-	schemaData.close();
-
-	if (error.error != QJsonParseError::NoError)
+	catch(const std::runtime_error& error)
 	{
-		// report to the user the failure and their locations in the document.
-		int errorLine(0), errorColumn(0);
-
-		for( int i=0, count=qMin( error.offset,schema.size()); i<count; ++i )
-		{
-			++errorColumn;
-			if(schema.at(i) == '\n' )
-			{
-				errorColumn = 0;
-				++errorLine;
-			}
-		}
-
-		std::stringstream sstream;
-		sstream << "ERROR: Json schema wrong: " << error.errorString().toStdString() << " at Line: " << errorLine << ", Column: " << errorColumn;
-
-		throw std::runtime_error(sstream.str());
+		throw std::runtime_error(error.what());
 	}

-	schemaJson = doc.object();
-
 	// collect all LED Devices
 	properties = schemaJson["properties"].toObject();
 	alldevices = LedDevice::getLedDeviceSchemas();
@@ -1139,7 +1116,7 @@ bool JsonProcessor::checkJson(const QJsonObject& message, const QString& schemaR
 	schemaChecker.setSchema(schemaJson.object());

 	// check the message
-	if (!schemaChecker.validate(message, ignoreRequired))
+	if (!schemaChecker.validate(message, ignoreRequired).first)
 	{
 		const QStringList & errors = schemaChecker.getMessages();
 		errorMessage = "{";
--- a/libsrc/utils/jsonschema/QJsonSchemaChecker.cpp
+++ b/libsrc/utils/jsonschema/QJsonSchemaChecker.cpp
@@ -5,6 +5,7 @@

 // Utils-Jsonschema includes
 #include <utils/jsonschema/QJsonSchemaChecker.h>
+#include <utils/jsonschema/QJsonUtils.h>

 QJsonSchemaChecker::QJsonSchemaChecker()
 {
@@ -25,11 +26,12 @@ bool QJsonSchemaChecker::setSchema(const QJsonObject & schema)
 	return true;
 }

-bool QJsonSchemaChecker::validate(const QJsonObject & value, bool ignoreRequired)
+QPair<bool, bool> QJsonSchemaChecker::validate(const QJsonObject & value, bool ignoreRequired)
 {
 	// initialize state
 	_ignoreRequired = ignoreRequired;
 	_error = false;
+	_schemaError = false;
 	_messages.clear();
 	_currentPath.clear();
 	_currentPath.append("[root]");
@@ -37,7 +39,27 @@ bool QJsonSchemaChecker::validate(const QJsonObject & value, bool ignoreRequired
 	// validate
 	validate(value, _qSchema);

-	return !_error;
+	return QPair<bool, bool>(!_error, !_schemaError);
+}
+
+QJsonObject QJsonSchemaChecker::getAutoCorrectedConfig(const QJsonObject& value, bool ignoreRequired)
+{
+	_ignoreRequired = ignoreRequired;
+	QStringList sequence = QStringList() << "remove" << "modify" << "create";
+	_error = false;
+	_schemaError = false;
+	_messages.clear();
+	_autoCorrected = value;
+
+	for(const QString &correct : sequence)
+	{
+		_correct = correct;
+		_currentPath.clear();
+		_currentPath.append("[root]");
+		validate(_autoCorrected, _qSchema);
+	}
+
+	return _autoCorrected;
 }

 void QJsonSchemaChecker::validate(const QJsonValue & value, const QJsonObject &schema)
@@ -48,15 +70,17 @@ void QJsonSchemaChecker::validate(const QJsonValue & value, const QJsonObject &s
 		QString attribute = i.key();
 		const QJsonValue & attributeValue = *i;

+		QJsonObject::const_iterator defaultValue = schema.find("default");
+
 		if (attribute == "type")
-			checkType(value, attributeValue);
+			checkType(value, attributeValue, (defaultValue != schema.end() ? defaultValue.value() : QJsonValue::Null));
 		else if (attribute == "properties")
 		{
 			if (value.isObject())
 				checkProperties(value.toObject(), attributeValue.toObject());
 			else
 			{
-				_error = true;
+				_schemaError = true;
 				setMessage("properties attribute is only valid for objects");
 				continue;
 			}
@@ -76,19 +100,19 @@ void QJsonSchemaChecker::validate(const QJsonValue & value, const QJsonObject &s
 			}
 			else
 			{
-				_error = true;
+				_schemaError = true;
 				setMessage("additional properties attribute is only valid for objects");
 				continue;
 			}
 		}
 		else if (attribute == "minimum")
-			checkMinimum(value, attributeValue);
+			checkMinimum(value, attributeValue, (defaultValue != schema.end() ? defaultValue.value() : QJsonValue::Null));
 		else if (attribute == "maximum")
-			checkMaximum(value, attributeValue);
+			checkMaximum(value, attributeValue, (defaultValue != schema.end() ? defaultValue.value() : QJsonValue::Null));
 		else if (attribute == "minLength")
-			checkMinLength(value, attributeValue);
+			checkMinLength(value, attributeValue, (defaultValue != schema.end() ? defaultValue.value() : QJsonValue::Null));
 		else if (attribute == "maxLength")
-			checkMaxLength(value, attributeValue);
+			checkMaxLength(value, attributeValue, (defaultValue != schema.end() ? defaultValue.value() : QJsonValue::Null));
 		else if (attribute == "items")
 		{
 			if (value.isArray())
@@ -101,13 +125,13 @@ void QJsonSchemaChecker::validate(const QJsonValue & value, const QJsonObject &s
 			}
 		}
 		else if (attribute == "minItems")
-			checkMinItems(value, attributeValue);
+			checkMinItems(value, attributeValue, (defaultValue != schema.end() ? defaultValue.value() : QJsonValue::Null));
 		else if (attribute == "maxItems")
-			checkMaxItems(value, attributeValue);
+			checkMaxItems(value, attributeValue, (defaultValue != schema.end() ? defaultValue.value() : QJsonValue::Null));
 		else if (attribute == "uniqueItems")
 			checkUniqueItems(value, attributeValue);
 		else if (attribute == "enum")
-			checkEnum(value, attributeValue);
+			checkEnum(value, attributeValue, (defaultValue != schema.end() ? defaultValue.value() : QJsonValue::Null));
 		else if (attribute == "required")
 			; // nothing to do. value is present so always oke
 		else if (attribute == "id")
@@ -118,7 +142,7 @@ void QJsonSchemaChecker::validate(const QJsonValue & value, const QJsonObject &s
 		else
 		{
 			// no check function defined for this attribute
-			_error = true;
+			_schemaError = true;
 			setMessage("No check function defined for attribute " + attribute);
 			continue;
 		}
@@ -135,7 +159,7 @@ const QStringList & QJsonSchemaChecker::getMessages() const
 	return _messages;
 }

-void QJsonSchemaChecker::checkType(const QJsonValue & value, const QJsonValue & schema)
+void QJsonSchemaChecker::checkType(const QJsonValue & value, const QJsonValue & schema, const QJsonValue & defaultValue)
 {
 	QString type = schema.toString();

@@ -145,7 +169,12 @@ void QJsonSchemaChecker::checkType(const QJsonValue & value, const QJsonValue &
 	else if (type == "number")
 		wrongType = !value.isDouble();
 	else if (type == "integer")
-		wrongType = (rint(value.toDouble()) != value.toDouble());
+	{
+		if (value.isDouble()) //check if value type not boolean (true = 1 && false = 0)
+			wrongType = (rint(value.toDouble()) != value.toDouble());
+		else
+			wrongType = true;
+	}
 	else if (type == "double")
 		wrongType = !value.isDouble();
 	else if (type == "boolean")
@@ -160,13 +189,16 @@ void QJsonSchemaChecker::checkType(const QJsonValue & value, const QJsonValue &
 		wrongType = !value.isString();
 	else if (type == "any")
 		wrongType = false;
-//	else
-//		assert(false);

 	if (wrongType)
 	{
 		_error = true;
-		setMessage(type + " expected");
+
+		if (_correct == "modify")
+			QJsonUtils::modify(_autoCorrected, _currentPath, defaultValue);
+
+		if (_correct == "")
+			setMessage(type + " expected");
 	}
 }

@@ -176,7 +208,7 @@ void QJsonSchemaChecker::checkProperties(const QJsonObject & value, const QJsonO
 	{
 		QString property = i.key();

-		const QJsonValue  & propertyValue = i.value();
+		const QJsonValue  & propertyValue = *i;

 		_currentPath.append("." + property);
 		QJsonObject::const_iterator required = propertyValue.toObject().find("required");
@@ -185,11 +217,19 @@ void QJsonSchemaChecker::checkProperties(const QJsonObject & value, const QJsonO
 		{
 			validate(value[property], propertyValue.toObject());
 		}
-		else if (required != propertyValue.toObject().end() && required.value().toBool() && !_ignoreRequired)
+		else if (required != propertyValue.toObject().end() && propertyValue.toObject().find("required").value().toBool() && !_ignoreRequired)
 		{
 			_error = true;
-			setMessage("missing member");
+
+			if (_correct == "create")
+				QJsonUtils::modify(_autoCorrected, _currentPath,  QJsonUtils::create(propertyValue, _ignoreRequired), property);
+
+			if (_correct == "")
+				setMessage("missing member");
 		}
+		else if (_correct == "create" && _ignoreRequired)
+			QJsonUtils::modify(_autoCorrected, _currentPath,  QJsonUtils::create(propertyValue, _ignoreRequired), property);
+
 		_currentPath.removeLast();
 	}
 }
@@ -208,7 +248,12 @@ void QJsonSchemaChecker::checkAdditionalProperties(const QJsonObject & value, co
 				if (schema.toBool() == false)
 				{
 					_error = true;
-					setMessage("no schema definition");
+
+					if (_correct == "remove")
+						QJsonUtils::modify(_autoCorrected, _currentPath);
+
+					if (_correct == "")
+						setMessage("no schema definition");
 				}
 			}
 			else
@@ -220,7 +265,7 @@ void QJsonSchemaChecker::checkAdditionalProperties(const QJsonObject & value, co
 	}
 }

-void QJsonSchemaChecker::checkMinimum(const QJsonValue & value, const QJsonValue & schema)
+void QJsonSchemaChecker::checkMinimum(const QJsonValue & value, const QJsonValue & schema, const QJsonValue & defaultValue)
 {
 	if (!value.isDouble())
 	{
@@ -233,11 +278,18 @@ void QJsonSchemaChecker::checkMinimum(const QJsonValue & value, const QJsonValue
 	if (value.toDouble() < schema.toDouble())
 	{
 		_error = true;
-		setMessage("value is too small (minimum=" + schema.toString() + ")");
+
+		if (_correct == "modify")
+			(defaultValue != QJsonValue::Null) ?
+			QJsonUtils::modify(_autoCorrected, _currentPath, defaultValue) :
+			QJsonUtils::modify(_autoCorrected, _currentPath, schema);
+
+		if (_correct == "")
+			setMessage("value is too small (minimum=" + QString::number(schema.toDouble()) + ")");
 	}
 }

-void QJsonSchemaChecker::checkMaximum(const QJsonValue & value, const QJsonValue & schema)
+void QJsonSchemaChecker::checkMaximum(const QJsonValue & value, const QJsonValue & schema, const QJsonValue & defaultValue)
 {
 	if (!value.isDouble())
 	{
@@ -250,11 +302,18 @@ void QJsonSchemaChecker::checkMaximum(const QJsonValue & value, const QJsonValue
 	if (value.toDouble() > schema.toDouble())
 	{
 		_error = true;
-		setMessage("value is too large (maximum=" + schema.toString() + ")");
+
+		if (_correct == "modify")
+			(defaultValue != QJsonValue::Null) ?
+			QJsonUtils::modify(_autoCorrected, _currentPath, defaultValue) :
+			QJsonUtils::modify(_autoCorrected, _currentPath, schema);
+
+		if (_correct == "")
+			setMessage("value is too large (maximum=" + QString::number(schema.toDouble()) + ")");
 	}
 }

-void QJsonSchemaChecker::checkMinLength(const QJsonValue & value, const QJsonValue & schema)
+void QJsonSchemaChecker::checkMinLength(const QJsonValue & value, const QJsonValue & schema, const QJsonValue & defaultValue)
 {
 	if (!value.isString())
 	{
@@ -267,11 +326,18 @@ void QJsonSchemaChecker::checkMinLength(const QJsonValue & value, const QJsonVal
 	if (value.toString().size() < schema.toInt())
 	{
 		_error = true;
-		setMessage("value is too short (minLength=" + schema.toString() + ")");
+
+		if (_correct == "modify")
+			(defaultValue != QJsonValue::Null) ?
+			QJsonUtils::modify(_autoCorrected, _currentPath, defaultValue) :
+			QJsonUtils::modify(_autoCorrected, _currentPath, schema);
+
+		if (_correct == "")
+			setMessage("value is too short (minLength=" + QString::number(schema.toInt()) + ")");
 	}
 }

-void QJsonSchemaChecker::checkMaxLength(const QJsonValue & value, const QJsonValue & schema)
+void QJsonSchemaChecker::checkMaxLength(const QJsonValue & value, const QJsonValue & schema, const QJsonValue & defaultValue)
 {
 	if (!value.isString())
 	{
@@ -284,7 +350,14 @@ void QJsonSchemaChecker::checkMaxLength(const QJsonValue & value, const QJsonVal
 	if (value.toString().size() > schema.toInt())
 	{
 		_error = true;
-		setMessage("value is too long (maxLength=" + schema.toString() + ")");
+
+		if (_correct == "modify")
+			(defaultValue != QJsonValue::Null) ?
+			QJsonUtils::modify(_autoCorrected, _currentPath, defaultValue) :
+			QJsonUtils::modify(_autoCorrected, _currentPath, schema);
+
+		if (_correct == "")
+			setMessage("value is too long (maxLength=" + QString::number(schema.toInt()) + ")");
 	}
 }

@@ -299,6 +372,11 @@ void QJsonSchemaChecker::checkItems(const QJsonValue & value, const QJsonObject
 	}

 	QJsonArray jArray = value.toArray();
+
+	if (_correct == "remove")
+		if (jArray.isEmpty())
+			QJsonUtils::modify(_autoCorrected, _currentPath);
+
 	for(int i = 0; i < jArray.size(); ++i)
 	{
 		// validate each item
@@ -308,7 +386,7 @@ void QJsonSchemaChecker::checkItems(const QJsonValue & value, const QJsonObject
 	}
 }

-void QJsonSchemaChecker::checkMinItems(const QJsonValue & value, const QJsonValue & schema)
+void QJsonSchemaChecker::checkMinItems(const QJsonValue & value, const QJsonValue & schema, const QJsonValue & defaultValue)
 {
 	if (!value.isArray())
 	{
@@ -318,17 +396,22 @@ void QJsonSchemaChecker::checkMinItems(const QJsonValue & value, const QJsonValu
 		return;
 	}

-	int minimum = schema.toInt();
-
 	QJsonArray jArray = value.toArray();
-	if (static_cast<int>(jArray.size()) < minimum)
+	if (jArray.size() < schema.toInt())
 	{
 		_error = true;
-		setMessage("array is too large (minimum=" + QString::number(minimum) + ")");
+
+		if (_correct == "modify")
+			(defaultValue != QJsonValue::Null) ?
+			QJsonUtils::modify(_autoCorrected, _currentPath, defaultValue) :
+			QJsonUtils::modify(_autoCorrected, _currentPath, schema);
+
+		if (_correct == "")
+			setMessage("array is too small (minimum=" + QString::number(schema.toInt()) + ")");
 	}
 }

-void QJsonSchemaChecker::checkMaxItems(const QJsonValue & value, const QJsonValue & schema)
+void QJsonSchemaChecker::checkMaxItems(const QJsonValue & value, const QJsonValue & schema, const QJsonValue & defaultValue)
 {
 	if (!value.isArray())
 	{
@@ -338,13 +421,18 @@ void QJsonSchemaChecker::checkMaxItems(const QJsonValue & value, const QJsonValu
 		return;
 	}

-	int maximum = schema.toInt();
-
 	QJsonArray jArray = value.toArray();
-	if (static_cast<int>(jArray.size()) > maximum)
+	if (jArray.size() > schema.toInt())
 	{
 		_error = true;
-		setMessage("array is too large (maximum=" + QString::number(maximum) + ")");
+
+		if (_correct == "modify")
+			(defaultValue != QJsonValue::Null) ?
+			QJsonUtils::modify(_autoCorrected, _currentPath, defaultValue) :
+			QJsonUtils::modify(_autoCorrected, _currentPath, schema);
+
+		if (_correct == "")
+			setMessage("array is too large (maximum=" + QString::number(schema.toInt()) + ")");
 	}
 }

@@ -362,6 +450,8 @@ void QJsonSchemaChecker::checkUniqueItems(const QJsonValue & value, const QJsonV
 	{
 		// make sure no two items are identical

+		bool removeDuplicates = false;
+
 		QJsonArray jArray = value.toArray();
 		for(int i = 0; i < jArray.size(); ++i)
 		{
@@ -371,14 +461,28 @@ void QJsonSchemaChecker::checkUniqueItems(const QJsonValue & value, const QJsonV
 				{
 					// found a value twice
 					_error = true;
-					setMessage("array must have unique values");
+					removeDuplicates = true;
+
+					if (_correct == "")
+						setMessage("array must have unique values");
 				}
 			}
 		}
+
+		if (removeDuplicates && _correct == "modify")
+		{
+			QJsonArray uniqueItemsArray;
+			
+			for(int i = 0; i < jArray.size(); ++i)
+				if (!uniqueItemsArray.contains(jArray[i]))
+					uniqueItemsArray.append(jArray[i]);
+			
+			QJsonUtils::modify(_autoCorrected, _currentPath, uniqueItemsArray);
+		}
 	}
 }

-void QJsonSchemaChecker::checkEnum(const QJsonValue & value, const QJsonValue & schema)
+void QJsonSchemaChecker::checkEnum(const QJsonValue & value, const QJsonValue & schema, const QJsonValue & defaultValue)
 {
 	if (schema.isArray())
 	{
@@ -395,7 +499,16 @@ void QJsonSchemaChecker::checkEnum(const QJsonValue & value, const QJsonValue &

 	// nothing found
 	_error = true;
-	QJsonDocument doc(schema.toArray());
-	QString strJson(doc.toJson(QJsonDocument::Compact));
-	setMessage("Unknown enum value (allowed values are: " + schema.toString() + strJson+ ")");
+
+	if (_correct == "modify")
+		(defaultValue != QJsonValue::Null) ?
+		QJsonUtils::modify(_autoCorrected, _currentPath, defaultValue) :
+		QJsonUtils::modify(_autoCorrected, _currentPath, schema.toArray().first());
+
+	if (_correct == "")
+	{
+		QJsonDocument doc(schema.toArray());
+		QString strJson(doc.toJson(QJsonDocument::Compact));
+		setMessage("Unknown enum value (allowed values are: " + strJson+ ")");
+	}
 }