mlsentiment: allow custom tokenizers (#1026)

* allow custom tokenizers

allows use of tokens object to specifiy custom tokenizers

updated mlsentiment to v2.0.0 (with temporary fix for comparative score applied)

updated documentation

* update unit tests

analysis/mlsentiment/locales/en-US/mlsentiment.html

@@ -4,14 +4,16 @@
     <h3>Outputs</h3>
     <dl class="message-properties">
         <dt>sentiment <span class="property-type">object</span></dt>
-        <dd>contains the resulting AFINN-111 sentiment.</dd>
+        <dd>contains the resulting AFINN-165 sentiment.</dd>
         <dt>sentiment.score <span class="property-type">number</span></dt>
         <dd>the sentiment score.</dd>
     </dl>
     <h3>Inputs</h3>
     <dl class="message-properties">
         <dt>words <span class="property-type">object</span></dt>
-        <dd>an object of words and scores to override or add words can be supplied - <code>{ word:score,... }</code>.</dd>
+        <dd>besides sending a raw string in msg.payload, an object of words and scores to override or add words can be supplied - <code>{ word:score,... }</code>.</dd>
+        <dt>tokens <span class="property-type">object</span></dt>
+        <dd>a `tokens` object allows custom tokenizers which may be required by some languages - <code>{ ['世界',...] }</code>.</dd>
     </dl>
     <dl class="message-properties">
         <dt>lang <span class="property-type">string</span></dt>

analysis/mlsentiment/locales/ja/mlsentiment.html

@@ -4,17 +4,19 @@
     <h3>出力</h3>
     <dl class="message-properties">
         <dt>sentiment <span class="property-type">オブジェクト</span></dt>
-        <dd>AFINN-111による感情分析の結果</dd>
+        <dd>AFINN-165による感情分析の結果</dd>
         <dt>sentiment.score <span class="property-type">数値</span></dt>
         <dd>感情分析スコア</dd>
     </dl>
     <h3>入力</h3>
     <dl class="message-properties">
-        <dt>overrides <span class="property-type">オブジェクト</span></dt>
+        <dt>words <span class="property-type">オブジェクト</span></dt>
         <dd>単語スコアの上書きをするためのオブジェクト - <code>{ word:score,... }</code></dd>
+        <dt>tokens <span class="property-type">object</span></dt>
+        <dd>一部の言語で必要となるカスタム トークナイザーが可能になります - <code>{ ['世界',...] }</code>.</dd>
     </dl>
     <h3>詳細</h3>
     <p>ゼロ以上のスコアはポジティブ、ゼロ以下はネガティブを意味します。</p>
     <p>スコアの範囲は通常-5から+5ですが、より大きかったり小さかったりすることもあります。</p>
     <p>詳細は<a href="https://github.com/marcellobarile/multilang-sentiment/blob/develop/README.md" target="_blank">the Multilang-Sentiment docs here</a>を参照してください。</p>
-</script>
+</script>

analysis/mlsentiment/mlsentiment.js

@@ -1,28 +1,25 @@
 
-module.exports = function(RED) {
+module.exports = function (RED) {
     "use strict";
     var multilangsentiment = require('multilang-sentiment');
 
     function MultiLangSentimentNode(n) {
-        RED.nodes.createNode(this,n);
+        RED.nodes.createNode(this, n);
         this.lang = n.lang;
-        this.property = n.property||"payload";
+        this.property = n.property || "payload";
         var node = this;
 
-        this.on("input", function(msg) {
+        this.on("input", function (msg) {
-            var value = RED.util.getMessageProperty(msg,node.property);
+            var value = RED.util.getMessageProperty(msg, node.property);
             if (value !== undefined) {
-                if (msg.hasOwnProperty("overrides")) {
+                multilangSentiment(value, node.lang || msg.lang || 'en', { 'words': msg.words || null, 'tokens': msg.tokens || null }, function (err, result) {
-                    msg.extras = msg.overrides;
-                    delete msg.overrides;
-                }
-                multilangsentiment(value, node.lang || msg.lang || 'en', {words: msg.extras || null}, function (err, result) {
                     msg.sentiment = result;
+                    msg.sentiment.comparative = msg.sentiment.score / msg.sentiment.tokens.length;      // temporarily addresses an issue in v2.0.0: https://github.com/marcellobarile/multilang-sentiment/issues/10
                     node.send(msg);
                 });
             }
             else { node.send(msg); } // If no matching property - just pass it on.
         });
     }
-    RED.nodes.registerType("mlsentiment",MultiLangSentimentNode);
+    RED.nodes.registerType("mlsentiment", MultiLangSentimentNode);
 }

analysis/mlsentiment/package.json

@@ -1,19 +1,25 @@
 {
-    "name"          : "node-red-node-multilang-sentiment",
+    "name": "node-red-node-multilang-sentiment",
-    "version"       : "0.1.0",
+    "version": "0.2.0",
-    "description"   : "A Node-RED node that uses the AFINN-165 wordlists for sentiment analysis of words translated into multiple languages including emojis.",
+    "description": "A Node-RED node that uses the AFINN-165 wordlists for sentiment analysis of words translated into multiple languages including emoji.",
-    "dependencies"  : {
+    "dependencies": {
-        "multilang-sentiment" : "^1.2.0"
+        "multilang-sentiment": "^2.0.0"
     },
-    "repository" : {
+    "repository": {
-        "type":"git",
+        "type": "git",
-        "url":"https://github.com/node-red/node-red-nodes.git",
+        "url": "https://github.com/node-red/node-red-nodes.git",
-        "directory" : "tree/master/analysis/mlsentiment"
+        "directory": "tree/master/analysis/mlsentiment"
     },
     "license": "Apache-2.0",
-    "keywords": [ "node-red", "sentiment", "anaylsis", "AFINN" ],
+    "keywords": [
-    "node-red" : {
+        "node-red",
-        "nodes" : {
+        "sentiment",
+        "anaylsis",
+        "AFINN",
+        "emoji"
+    ],
+    "node-red": {
+        "nodes": {
             "mlsentiment": "mlsentiment.js"
         }
     },
@@ -21,5 +27,8 @@
         "name": "Dave Conway-Jones",
         "email": "ceejay@vnet.ibm.com",
         "url": "http://nodered.org"
-    }
+    },
-}
+    "contributors" : [
+        "Chuan Khoo <khoo.chuan@gmail.com> (https://chuank.com)"
+    ]
+}

test/analysis/mlsentiment/mlsentiment_spec.js

@@ -18,118 +18,118 @@ var should = require("should");
 var sentimentNode = require("../../../analysis/mlsentiment/mlsentiment.js");
 var helper = require("node-red-node-test-helper");
 
-describe('mlsentiment Node', function() {
+describe('mlsentiment Node', function () {
 
-    before(function(done) {
+    before(function (done) {
         helper.startServer(done);
     });
 
-    after(function(done) {
+    after(function (done) {
         helper.stopServer(done);
     });
 
-    afterEach(function() {
+    afterEach(function () {
         helper.unload();
     });
 
-    it('should be loaded', function(done) {
+    it('should be loaded', function (done) {
-        var flow = [{id:"sentimentNode1", type:"mlsentiment", name: "sentimentNode" }];
+        var flow = [{ id: "sentimentNode1", type: "mlsentiment", name: "sentimentNode" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var sentimentNode1 = helper.getNode("sentimentNode1");
             sentimentNode1.should.have.property('name', 'sentimentNode');
             done();
         });
     });
 
-    it('should pass on msg if no payload', function(done) {
+    it('should pass on msg if no payload', function (done) {
-        var flow = [{id:"jn1",type:"mlsentiment",wires:[["jn2"]]},
+        var flow = [{ id: "jn1", type: "mlsentiment", wires: [["jn2"]] },
-                    {id:"jn2", type:"helper"}];
+        { id: "jn2", type: "helper" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var jn1 = helper.getNode("jn1");
             var jn2 = helper.getNode("jn2");
-            jn2.on("input", function(msg) {
+            jn2.on("input", function (msg) {
                 msg.should.not.have.property('sentiment');
                 msg.topic.should.equal("pass on");
                 done();
             });
             var testString = 'good, great, best, brilliant';
-            jn1.receive({topic:"pass on"});
+            jn1.receive({ topic: "pass on" });
         });
     });
 
-    it('should add a positive score for good words', function(done) {
+    it('should add a positive score for good words', function (done) {
-        var flow = [{id:"jn1",type:"mlsentiment",wires:[["jn2"]]},
+        var flow = [{ id: "jn1", type: "mlsentiment", wires: [["jn2"]] },
-                    {id:"jn2", type:"helper"}];
+        { id: "jn2", type: "helper" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var jn1 = helper.getNode("jn1");
             var jn2 = helper.getNode("jn2");
-            jn2.on("input", function(msg) {
+            jn2.on("input", function (msg) {
                 try {
                     msg.should.have.property('sentiment');
                     msg.sentiment.should.have.property('score');
                     msg.sentiment.score.should.be.a.Number();
                     msg.sentiment.score.should.be.above(10);
                     done();
-                } catch(err) {
+                } catch (err) {
                     done(err);
                 }
             });
             var testString = 'good, great, best, brilliant';
-            jn1.receive({payload:testString});
+            jn1.receive({ payload: testString });
         });
     });
 
-    it('should add a positive score for good words (in French)', function(done) {
+    it('should add a positive score for good words (in French)', function (done) {
-        var flow = [{id:"jn1",type:"mlsentiment",wires:[["jn2"]],lang:"fr"},
+        var flow = [{ id: "jn1", type: "mlsentiment", wires: [["jn2"]], lang: "fr" },
-                    {id:"jn2", type:"helper"}];
+        { id: "jn2", type: "helper" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var jn1 = helper.getNode("jn1");
             var jn2 = helper.getNode("jn2");
-            jn2.on("input", function(msg) {
+            jn2.on("input", function (msg) {
                 try {
                     msg.should.have.property('sentiment');
                     msg.sentiment.should.have.property('score');
                     msg.sentiment.score.should.be.a.Number();
                     msg.sentiment.score.should.be.above(5);
                     done();
-                } catch(err) {
+                } catch (err) {
                     done(err);
                 }
             });
             var testString = 'bon, belle, don du ciel, brillant';
-            jn1.receive({payload:testString});
+            jn1.receive({ payload: testString });
         });
     });
 
-    it('should add a positive score for good words - alternative property', function(done) {
+    it('should add a positive score for good words - alternative property', function (done) {
-        var flow = [{id:"jn1",type:"mlsentiment",property:"foo",wires:[["jn2"]]},
+        var flow = [{ id: "jn1", type: "mlsentiment", property: "foo", wires: [["jn2"]] },
-                    {id:"jn2", type:"helper"}];
+        { id: "jn2", type: "helper" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var jn1 = helper.getNode("jn1");
             var jn2 = helper.getNode("jn2");
-            jn2.on("input", function(msg) {
+            jn2.on("input", function (msg) {
                 try {
                     msg.should.have.property('sentiment');
                     msg.sentiment.should.have.property('score');
                     msg.sentiment.score.should.be.a.Number();
                     msg.sentiment.score.should.be.above(10);
                     done();
-                } catch(err) {
+                } catch (err) {
                     done(err);
                 }
             });
             var testString = 'good, great, best, brilliant';
-            jn1.receive({foo:testString});
+            jn1.receive({ foo: testString });
         });
     });
 
-    it('should add a negative score for bad words', function(done) {
+    it('should add a negative score for bad words', function (done) {
-        var flow = [{id:"jn1",type:"mlsentiment",wires:[["jn2"]]},
+        var flow = [{ id: "jn1", type: "mlsentiment", wires: [["jn2"]] },
-                    {id:"jn2", type:"helper"}];
+        { id: "jn2", type: "helper" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var jn1 = helper.getNode("jn1");
             var jn2 = helper.getNode("jn2");
-            jn2.on("input", function(msg) {
+            jn2.on("input", function (msg) {
                 msg.should.have.property('sentiment');
                 msg.sentiment.should.have.property('score');
                 msg.sentiment.score.should.be.a.Number();
@@ -137,17 +137,17 @@ describe('mlsentiment Node', function() {
                 done();
             });
             var testString = 'bad, horrible, negative, awful';
-            jn1.receive({payload:testString});
+            jn1.receive({ payload: testString });
         });
     });
 
-    it('should add a negative score for bad words - alternative property', function(done) {
+    it('should add a negative score for bad words - alternative property', function (done) {
-        var flow = [{id:"jn1",type:"mlsentiment",property:"foo",wires:[["jn2"]]},
+        var flow = [{ id: "jn1", type: "mlsentiment", property: "foo", wires: [["jn2"]] },
-                    {id:"jn2", type:"helper"}];
+        { id: "jn2", type: "helper" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var jn1 = helper.getNode("jn1");
             var jn2 = helper.getNode("jn2");
-            jn2.on("input", function(msg) {
+            jn2.on("input", function (msg) {
                 msg.should.have.property('sentiment');
                 msg.sentiment.should.have.property('score');
                 msg.sentiment.score.should.be.a.Number();
@@ -155,17 +155,17 @@ describe('mlsentiment Node', function() {
                 done();
             });
             var testString = 'bad, horrible, negative, awful';
-            jn1.receive({foo:testString});
+            jn1.receive({ foo: testString });
         });
     });
 
-    it('should allow you to override word scoring', function(done) {
+    it('should allow you to override word scoring', function (done) {
-        var flow = [{id:"jn1",type:"mlsentiment",wires:[["jn2"]]},
+        var flow = [{ id: "jn1", type: "mlsentiment", wires: [["jn2"]] },
-                    {id:"jn2", type:"helper"}];
+        { id: "jn2", type: "helper" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var jn1 = helper.getNode("jn1");
             var jn2 = helper.getNode("jn2");
-            jn2.on("input", function(msg) {
+            jn2.on("input", function (msg) {
                 msg.should.have.property('sentiment');
                 msg.sentiment.should.have.property('score');
                 msg.sentiment.score.should.be.a.Number();
@@ -173,18 +173,18 @@ describe('mlsentiment Node', function() {
                 done();
             });
             var testString = 'sick, wicked';
-            var overrides = {'sick': 10, 'wicked': 10 };
+            var wordOverrides = { 'sick': 10, 'wicked': 10 };
-            jn1.receive({payload:testString,overrides:overrides});
+            jn1.receive({ payload: testString, words: wordOverrides });
         });
     });
 
-    it('should allow you to override word scoring - alternative property', function(done) {
+    it('should allow you to override word scoring - alternative property', function (done) {
-        var flow = [{id:"jn1",type:"mlsentiment",property:"foo",wires:[["jn2"]]},
+        var flow = [{ id: "jn1", type: "mlsentiment", property: "foo", wires: [["jn2"]] },
-                    {id:"jn2", type:"helper"}];
+        { id: "jn2", type: "helper" }];
-        helper.load(sentimentNode, flow, function() {
+        helper.load(sentimentNode, flow, function () {
             var jn1 = helper.getNode("jn1");
             var jn2 = helper.getNode("jn2");
-            jn2.on("input", function(msg) {
+            jn2.on("input", function (msg) {
                 msg.should.have.property('sentiment');
                 msg.sentiment.should.have.property('score');
                 msg.sentiment.score.should.be.a.Number();
@@ -192,8 +192,46 @@ describe('mlsentiment Node', function() {
                 done();
             });
             var testString = 'sick, wicked';
-            var overrides = {'sick': 10, 'wicked': 10 };
+            var wordOverrides = { 'sick': 10, 'wicked': 10 };
-            jn1.receive({foo:testString,overrides:overrides});
+            jn1.receive({ foo: testString, words: wordOverrides });
+        });
+    });
+
+    it('should allow you to use custom tokens', function (done) {
+        var flow = [{ id: "jn1", type: "mlsentiment", wires: [["jn2"]] },
+        { id: "jn2", type: "helper" }];
+        helper.load(sentimentNode, flow, function () {
+            var jn1 = helper.getNode("jn1");
+            var jn2 = helper.getNode("jn2");
+            jn2.on("input", function (msg) {
+                msg.should.have.property('sentiment');
+                msg.sentiment.should.have.property('score');
+                msg.sentiment.score.should.be.a.Number();
+                msg.sentiment.score.should.equal(-3);
+                done();
+            });
+            var testString = '世界就是一个疯子的囚笼';
+            var tokenOverrides = ['世界', '就', '是', '一个', '疯子', '的', '囚笼'];
+            jn1.receive({ payload: testString, tokens: tokenOverrides });
+        });
+    });
+
+    it('should allow you to use custom tokens - alternative property', function (done) {
+        var flow = [{ id: "jn1", type: "mlsentiment", property: "foo", wires: [["jn2"]] },
+        { id: "jn2", type: "helper" }];
+        helper.load(sentimentNode, flow, function () {
+            var jn1 = helper.getNode("jn1");
+            var jn2 = helper.getNode("jn2");
+            jn2.on("input", function (msg) {
+                msg.should.have.property('sentiment');
+                msg.sentiment.should.have.property('score');
+                msg.sentiment.score.should.be.a.Number();
+                msg.sentiment.score.should.equal(-3);
+                done();
+            });
+            var testString = '世界就是一个疯子的囚笼';
+            var tokenOverrides = ['世界', '就', '是', '一个', '疯子', '的', '囚笼'];
+            jn1.receive({ foo: testString, tokens: tokenOverrides });
         });
     });