allow custom tokenizers

allows use of tokens object to specifiy custom tokenizers

updated mlsentiment to v2.0.0 (with temporary fix for comparative score applied)

updated documentation

analysis/mlsentiment/locales/en-US/mlsentiment.html

@@ -4,14 +4,16 @@
     <h3>Outputs</h3>
     <dl class="message-properties">
         <dt>sentiment <span class="property-type">object</span></dt>
-        <dd>contains the resulting AFINN-111 sentiment.</dd>
+        <dd>contains the resulting AFINN-165 sentiment.</dd>
         <dt>sentiment.score <span class="property-type">number</span></dt>
         <dd>the sentiment score.</dd>
     </dl>
     <h3>Inputs</h3>
     <dl class="message-properties">
         <dt>words <span class="property-type">object</span></dt>
-        <dd>an object of words and scores to override or add words can be supplied - <code>{ word:score,... }</code>.</dd>
+        <dd>besides sending a raw string in msg.payload, an object of words and scores to override or add words can be supplied - <code>{ word:score,... }</code>.</dd>
+        <dt>tokens <span class="property-type">object</span></dt>
+        <dd>a `tokens` object allows custom tokenizers which may be required by some languages - <code>{ ['世界',...] }</code>.</dd>
     </dl>
     <dl class="message-properties">
         <dt>lang <span class="property-type">string</span></dt>

analysis/mlsentiment/locales/ja/mlsentiment.html

@@ -4,14 +4,16 @@
     <h3>出力</h3>
     <dl class="message-properties">
         <dt>sentiment <span class="property-type">オブジェクト</span></dt>
-        <dd>AFINN-111による感情分析の結果</dd>
+        <dd>AFINN-165による感情分析の結果</dd>
         <dt>sentiment.score <span class="property-type">数値</span></dt>
         <dd>感情分析スコア</dd>
     </dl>
     <h3>入力</h3>
     <dl class="message-properties">
-        <dt>overrides <span class="property-type">オブジェクト</span></dt>
+        <dt>words <span class="property-type">オブジェクト</span></dt>
         <dd>単語スコアの上書きをするためのオブジェクト - <code>{ word:score,... }</code></dd>
+        <dt>tokens <span class="property-type">object</span></dt>
+        <dd>一部の言語で必要となるカスタム トークナイザーが可能になります - <code>{ ['世界',...] }</code>.</dd>
     </dl>
     <h3>詳細</h3>
     <p>ゼロ以上のスコアはポジティブ、ゼロ以下はネガティブを意味します。</p>

analysis/mlsentiment/mlsentiment.js

@@ -16,8 +16,9 @@ module.exports = function(RED) {
                     msg.extras = msg.overrides;
                     delete msg.overrides;
                 }
-                multilangsentiment(value, node.lang || msg.lang || 'en', {words: msg.extras || null}, function (err, result) {
+                multilangSentiment(value, node.lang || msg.lang || 'en', { 'words': msg.words || null, 'tokens': msg.tokens || null }, function (err, result) {
                     msg.sentiment = result;
+                    msg.sentiment.comparative = msg.sentiment.score / msg.sentiment.tokens.length;      // temporarily addresses an issue in v2.0.0: https://github.com/marcellobarile/multilang-sentiment/issues/10
                     node.send(msg);
                 });
             }

analysis/mlsentiment/package.json

@@ -1,9 +1,9 @@
 {
     "name": "node-red-node-multilang-sentiment",
-    "version"       : "0.1.0",
+    "version": "0.2.0",
-    "description"   : "A Node-RED node that uses the AFINN-165 wordlists for sentiment analysis of words translated into multiple languages including emojis.",
+    "description": "A Node-RED node that uses the AFINN-165 wordlists for sentiment analysis of words translated into multiple languages including emoji.",
     "dependencies": {
-        "multilang-sentiment" : "^1.2.0"
+        "multilang-sentiment": "^2.0.0"
     },
     "repository": {
         "type": "git",
@@ -11,7 +11,13 @@
         "directory": "tree/master/analysis/mlsentiment"
     },
     "license": "Apache-2.0",
-    "keywords": [ "node-red", "sentiment", "anaylsis", "AFINN" ],
+    "keywords": [
+        "node-red",
+        "sentiment",
+        "anaylsis",
+        "AFINN",
+        "emoji"
+    ],
     "node-red": {
         "nodes": {
             "mlsentiment": "mlsentiment.js"
@@ -21,5 +27,8 @@
         "name": "Dave Conway-Jones",
         "email": "ceejay@vnet.ibm.com",
         "url": "http://nodered.org"
-    }
+    },
+    "contributors" : [
+        "Chuan Khoo <khoo.chuan@gmail.com> (https://chuank.com)"
+    ]
 }