1. Using Node.js to
Build Great
Streaming Services
@sh1mmer

2. Tom Hughes-Croucher
@sh1mmer

3. @sh1mmer

4. @sh1mmer

5. Scalable Server-Side Code with JavaScript
Node Up and Running
Tom Hughes-Croucher
http://ofps.oreilly.com/titles/9781449398583/
http://shop.oreilly.com/product/0636920015956.do
@sh1mmer

6. A story
@sh1mmer

7. factory
@sh1mmer

8. @sh1mmer

9. @sh1mmer

10. var http = require('http');
var server = http.createServer();
server.listen(8000);
server.on('request', function(req,res) {
var doodad = new Doodad();
res.writeHead(200,{'Content-Type':'text/doodad'});
res.end(doodad);
});
@sh1mmer

11. Making Doodads
@sh1mmer

12. Knobs
Screens Antennas
& buttons
@sh1mmer

13. @sh1mmer

14. @sh1mmer

15. @sh1mmer

16. @sh1mmer

17. @sh1mmer

18. var http = require('http'), fs = require('fs');
var server = http.createServer();
server.listen(8000);
server.on('request', function(req,res) {
var screen = fs.readFileSync('/factory1/screen');
var dial = fs.readFileSync('/factory2/dial');
var doodad = new Doodad(screen, dial);
res.writeHead(200,{'Content-Type':'text/doodad'});
res.end(doodad);
});
@sh1mmer

19. Improving the factory
with streaming
@sh1mmer

20. @sh1mmer

21. @sh1mmer

22. @sh1mmer

23. var http = require('http');
var server = http.createServer();
server.listen(8000);
server.on('request', function(req,res) {
var parts = 0;
http.get('factory1/screen/', function(res) {
var screen = "";
res.on('data', function(d) { screen += d });
res.on('end', finish);
parts++;
};
http.get('factory2/dial', function(res) {
var dial = ""
res.on('data', function(d) { dial += d });
res.on('end', finish);
parts++;
}
function finish() {
if(parts == 2) {
var doodad = new Doodad(screen, dial);
res.writeHead(200,{'Content-Type':'text/doodad'});
res.end(doodad);
}
}
});
@sh1mmer

24. Streams API
@sh1mmer

25. Streams
• Readable • Writable
• 'data' event • write()
• 'end' event • end()
• pause()
• resume() • 'drain' event
• destroy() • destroy()
• pipe() • destroySoon()
@sh1mmer

26. var stream = require('stream');
var s = new stream.Stream(); //I am an EventEmitter
s.readable = true; //now I implement read API
s.emit('data', "my data");
s.emit('end'); //I'm done sending data
@sh1mmer

27. var stream = require('stream');
var s = new stream.Stream(); //I am an EventEmitter
s.writable = true; //now I implement write API
s.data = "";
s.write = function(d) {
s.data += d;
};
s.end = function() {
if(s.data) { console.log(s.data) }
};
s.destroy = function() {
s.writable = false;
}
@sh1mmer

28. The too many parts
problem and why
phone calls don't work
@sh1mmer

29. @sh1mmer

30. nextTick
Event Loop Fail
nextTick
Run stuff
TCP
Conn node.cc
FS
Read
add-ons v8
TCP
Conn
Timeout libuv
TCP
Interval
Conn
libev iocp
FS
Timeout Read
FS
Read

31. stream.pause()

32. nextTick
Event Loop Fail
nextTick
stream.pause()
TCP
Conn node.cc
FS
Read
add-ons v8
TCP
Conn
Timeout libuv
TCP
Interval
Conn
libev iocp
FS
Timeout Read
FS
Read

33. Managing Backpressure
• Manage Buffer sizes
• Use pause() / resume() to control back
pressure
• Assume that 'data' event may happen after
pause
• Remember node buffer -> kernel buffer
@sh1mmer

34. //manage Node's write Buffers
if(socket.bufferSize > maxSafe) {
writeStream.pause();
}
//Manage RS read stream Buffers
fs.createReadStream('./path/to/file', {
flags: 'r',
encoding: null,
fd: null,
mode: 0666,
bufferSize: 64 * 1024
});
@sh1mmer

35. New API
@sh1mmer

36. New Streams TBD
• Writable streams
• Same
• Readable (https://github.com/isaacs/readable-stream)
• configurable stream buffer waterline
• 'readable' event
• read(len) -> returns data
• Pipe
• Subscribers must implement readable stream
• No more direct access with Pipe

37. @sh1mmer

38. Types of Streaming
@sh1mmer

39. Simplex
@sh1mmer

40. var fs = require('fs');
var rs = fs.createReadStream('/my/file/path');
rs.on('data', function(d) {
console.log(d);
});
@sh1mmer

41. Throughput
@sh1mmer

42. var fs = require('fs');
var rs = fs.createReadStream('/my/file/path');
rs.pipe(process.stdout);
@sh1mmer

43. Duplex
@sh1mmer

44. var bot1 = new chatterBot();
var bot2 = new chatterBot();
bot1.pipe(bot2).pipe(bot1);
@sh1mmer

45. Libraries
@sh1mmer

46. JSONStream
@sh1mmer

47. {"total_rows":129,"offset":0,"rows":[
{ "id":"change1_0.6995461115147918"
, "key":"change1_0.6995461115147918"
, "value":{"rev":"1-e240bae28c7bb3667f02760f6398d508"}
, "doc":{
"_id": "change1_0.6995461115147918"
, "_rev": "1-e240bae28c7bb3667f02760f6398d508","hello":1}
},
{ "id":"change2_0.6995461115147918"
, "key":"change2_0.6995461115147918"
, "value":{"rev":"1-13677d36b98c0c075145bb8975105153"}
, "doc":{
"_id":"change2_0.6995461115147918"
, "_rev":"1-13677d36b98c0c075145bb8975105153"
, "hello":2
}
},
]}
@sh1mmer

48. var stream = JSONStream.parse(['rows', true, 'doc']) //rows,
ANYTHING, doc
stream.on('data', function(data) {
console.log('received:', data);
});
stream.on('root', function(root, count) {
if (!count) {
console.log('no matches found:', root);
}
});
@sh1mmer

49. node-libexpat
@sh1mmer

50. Application
@sh1mmer

51. Questions?
@sh1mmer