a blog for those who code

Thursday 29 September 2016

Real Time Twitter Trends With RethinkDB and Nodejs

In this post we will be discussing about getting real time twitter trends with RethinkDB and Nodejs. In our previous post Getting Started with RethinkDB in Nodejs we got to know how RethinkDB will be best suited when we have real time data coming in and we want that data to continuously refresh in our front end for the users to see.

Introduction


We will be reusing the code from Fetching Twitter Trends by places in Node.js where we had created an application in Nodejs and made requests to Twitter's Rest Service to get the current trends by places.

One more thing to note that whenever there is a change in the RethinkDB database we have to notify the frontend to get the new values. For this thing we have to use WebSocket. If you are not familiar with WebSocket in Node.js, I would recommend you to go through How to create WebSocket Server in Node.js and How to create WebSocket Client in Node.js.

Coding 


Calling Twitter to get Twitter Trends every 5 minutes : In the below code we are calling the twitter Places API to get the trends of India every 5 minutes. We are deleting the RethinkDB table and then inserting the new values. Just for the sake of this tutorial we are not checking that the values existed or not and thus deleting and inserting all values. We are only getting first 5 twitter trends and not more than that.

var https = require('https');
var headers = {
  'User-Agent': 'Coding Defined',
  Authorization: 'Bearer ' + require('./oauth.json').access_token
};
var rethinkDb = require('rethinkdbdash')();

function callTwitter(options, callback){
  https.get(options, function(response) {
    jsonHandler(response, callback);
  }).on('error', function(e) {
    console.log('Error : ' + e.message);
  })
}

var trendOptions = {
  host: 'api.twitter.com',
  path: '/1.1/trends/place.json?id=23424848',
  headers: headers
}

function jsonHandler(response, callback) {
  var json = '';
  response.setEncoding('utf8');
  if(response.statusCode === 200) {
    response.on('data', function(chunk) {
      json += chunk;
    }).on('end', function() {
rethinkDb.db('realTime').table('twitterTrend').delete().run();
callback(JSON.parse(json));
    });
  } else {
    console.log('Error : ' + reseponse.statusCode);
  }
}

var minutes = 5, the_interval = minutes * 60 * 1000;
setInterval(function() {
  callTwitter(trendOptions, function(trendsArray) {
    var count = 0;
    trendsArray[0].trends.forEach(function(trend) {
      count++;
      if(count > 5) {
return;
      }
      console.log(count + ". " + trend.name);
      rethinkDb.db('realTime').table('twitterTrend')
      .insert({
count: count,
name: trend.name
      })
      .run().then(function(response){
        console.log('Inserted');
      }).error(function(err){
 console.log('Error ' + err);
      });
    })
  });
}, the_interval);

WebSocket Server : In this we are opening the server on port 8076 and then subscribing to the real-time feeds using changes() function. Whenever there is a change our

var WebSocket = require('ws');
var WebSocketServer = WebSocket.Server;
var server = new WebSocketServer({port:8076});
var rethinkDb = require('rethinkdbdash')();

server.on('connection', function(socket) {
  rethinkDb.db('realTime').table('twitterTrend')
  .changes()
  .run().then(function(cursor){
    cursor.on("data", function(result) {
      console.log(result);
      socket.send(JSON.stringify(result));
    });
  }).error(function(err){
    console.log('Error ' + err);
  });

  socket.on('close', function(msg, disconnect) {
    console.log(msg + ' ' + disconnect);
  });
});

WebSocket Client : In this we will be listening to the WebSocket Server and log the responses.

var WebSocket = require('ws');
var ws = new WebSocket("ws://localhost:8076");

ws.on('open', function(message) {
  console.log('Received: ' + message);
});

ws.on('message', function(message) {
  console.log('Received: ' + message);
});

ws.on('close', function(code) {
  console.log('Disconnected: ' + code);
});

ws.on('error', function(error) {
  console.log('Error: ' + error.code);
});

WebSocket Client HTML File : In this we are getting the data on WebSocket onmessage event and then showing that on to the front-end.

<html>
  <head></head>
  <body>
    <h1> Latest Tweets </h1>
    <br/>
    <div id='1'></div>
    <div id='2'></div>
    <div id='3'></div>
    <div id='4'></div>
    <div id='5'></div>
    <div id='out'></div>
    <script type="text/javascript">
      (function () {
var ws = new WebSocket("ws://localhost:8076"),
var out = document.getElementById('out');

ws.onmessage = function(e) {
  var text = JSON.parse(e.data);
          if(text.new_val !== null && text.new_val.count !== undefined) {
    document.getElementById(text.new_val.count).innerHTML = '<b>' + text.new_val.count + '</b> : ' + text.new_val.name;
   }
}
        ws.onclose = function(e) {
  out.innerHTML += e.type;
};

ws.onerror = function(e) {
  out.innerHTML += e.data;
};

ws.onopen = function(e) {
  out.innerHTML += e.data;
};
      }());
    </script>
  </body>
</html>

Output : You can see the output in the below video



Please Like and Share the CodingDefined Blog, if you find it interesting and helpful.

No comments:

Post a Comment