Merge pull request #315 from vohof/feat/conserve

impose size restrictions when fetching link

package.json

@@ -28,14 +28,15 @@
     "bcrypt-nodejs": "0.0.3",
     "cheerio": "^0.17.0",
     "commander": "^2.3.0",
+    "event-stream": "^3.1.7",
     "express": "^4.9.5",
     "lodash": "~2.4.1",
     "mkdirp": "^0.5.0",
     "moment": "~2.7.0",
     "read": "^1.0.5",
+    "request": "^2.51.0",
     "slate-irc": "~0.7.3",
-    "socket.io": "~1.0.6",
+    "socket.io": "~1.0.6"
-    "superagent": "^0.18.2"
   },
   "devDependencies": {
     "grunt": "~0.4.5",

src/plugins/irc-events/link.js

@@ -1,8 +1,9 @@
 var _ = require("lodash");
 var cheerio = require("cheerio");
 var Msg = require("../../models/msg");
-var request = require("superagent");
+var request = require("request");
 var Helper = require("../../helper");
+var es = require('event-stream');
 
 module.exports = function(irc, network) {
 	var client = this;
@@ -60,7 +61,7 @@ function parse(msg, url, res, client) {
 
 	switch (res.type) {
 	case "text/html":
-		var $ = cheerio.load(res.res.text);
+		var $ = cheerio.load(res.text);
 		toggle.type = "link";
 		toggle.head = $("title").text();
 		toggle.body =
@@ -89,9 +90,35 @@ function parse(msg, url, res, client) {
 
 function fetch(url, cb) {
 	var req = request.get(url);
-	req.end(function(e, res) {
+	var length = 0;
-		if (res) {
+	var limit = 1024 * 10;
-			cb(res);
+	req
-		}
+		.on('response', function(res) {
-	});
+			if (!(/(text\/html|application\/json)/.test(res.headers['content-type']))) {
+			  res.req.abort();
+			}
+		})
+		.on('error', function() {})
+		.pipe(es.map(function(data, next) {
+			length += data.length;
+			if (length > limit) {
+				req.response.req.abort();
+			}
+			next(null, data);
+		}))
+		.pipe(es.wait(function(err, data) {
+			if (err) return;
+			var body;
+			try {
+				body = JSON.parse(data);
+			} catch(e) {
+				body = {};
+			}
+			data = {
+				text: data,
+				body: body,
+				type: req.response.headers['content-type'].split(/ *; */).shift()
+			};
+			cb(data);
+		}));
 }