From c14c2175731f272d69822391445903dfeb048514 Mon Sep 17 00:00:00 2001 From: Nik Petersen Date: Mon, 2 Jan 2012 17:06:00 -0800 Subject: [PATCH 01/15] Updated server.coffee with content-encoding header support --- server.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server.coffee b/server.coffee index 7b903df..66c41b3 100644 --- a/server.coffee +++ b/server.coffee @@ -125,7 +125,10 @@ server = Http.createServer (req, resp) -> 'content-length' : content_length 'Camo-Host' : camo_hostname 'X-Content-Type-Options' : 'nosniff' - + + if srcResp.headers['content-encoding'] + newHeaders['content-encoding'] = srcResp.headers['content-encoding'] + srcResp.on 'end', -> finish resp From 496c5d3f86d5b8e733aff3c5a61a21da0a695bbc Mon Sep 17 00:00:00 2001 From: Nik Petersen Date: Mon, 2 Jan 2012 17:07:33 -0800 Subject: [PATCH 02/15] Updated server.js with content-encoding support --- server.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server.js b/server.js index 46b7514..5a5174e 100644 --- a/server.js +++ b/server.js @@ -119,6 +119,11 @@ 'Camo-Host': camo_hostname, 'X-Content-Type-Options': 'nosniff' }; + + if (srcResp.headers['content-encoding']) { + newHeaders['content-encoding'] = srcResp.headers['content-encoding']; + } + srcResp.on('end', function() { return finish(resp); }); From cd98744850536176413b92642c60bc6e2b1ebec7 Mon Sep 17 00:00:00 2001 From: Dale Hui Date: Mon, 9 Jan 2012 22:12:26 -0800 Subject: [PATCH 03/15] Support HTTP 301s --- server.js | 144 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 65 deletions(-) diff --git a/server.js b/server.js index 5a5174e..eddd833 100644 --- a/server.js +++ b/server.js @@ -11,6 +11,7 @@ shared_key = process.env.CAMO_KEY || '0x24FEEDFACEDEADBEEFCAFE'; camo_hostname = process.env.CAMO_HOSTNAME || "unknown"; logging_enabled = process.env.CAMO_LOGGING_ENABLED || "disabled"; + max_redirects = process.env.CAMO_MAX_REDIRECTS || 10; log = function(msg) { if (logging_enabled !== "disabled") { console.log("--------------------------------------------"); @@ -45,6 +46,83 @@ return buf.toString(); } }; + process_url = function(url, transferred_headers, resp, remaining_redirects) { + if ((url.host != null) && !url.host.match(RESTRICTED_IPS)) { + if (url.host.match(EXCLUDED_HOSTS)) { + return four_oh_four(resp, "Hitting excluded hostnames"); + } + var src = Http.createClient(url.port || 80, url.hostname); + src.on('error', function(error) { + return four_oh_four(resp, "Client Request error " + error.stack); + }); + var query_path = url.pathname; + if (url.query != null) { + query_path += "?" + url.query; + } + transferred_headers.host = url.host; + log(transferred_headers); + var srcReq = src.request('GET', query_path, transferred_headers); + srcReq.on('response', function(srcResp) { + var content_length, newHeaders; + var is_finished = true; + log(srcResp.headers); + content_length = srcResp.headers['content-length']; + if (content_length > 5242880) { + return four_oh_four(resp, "Content-Length exceeded"); + } else { + newHeaders = { + 'expires': srcResp.headers['expires'], + 'content-type': srcResp.headers['content-type'], + 'cache-control': srcResp.headers['cache-control'], + 'content-length': content_length, + 'Camo-Host': camo_hostname, + 'X-Content-Type-Options': 'nosniff' + }; + if (srcResp.headers['content-encoding']) { + newHeaders['content-encoding'] = srcResp.headers['content-encoding']; + } + srcResp.on('end', function() { + if (is_finished) { + return finish(resp); + } + }); + srcResp.on('error', function() { + if (is_finished) { + return finish(resp); + } + }); + switch (srcResp.statusCode) { + case 200: + if (newHeaders['content-type'] && newHeaders['content-type'].slice(0, 5) !== 'image') { + return four_oh_four(resp, "Non-Image content-type returned"); + } + log(newHeaders); + resp.writeHead(srcResp.statusCode, newHeaders); + return srcResp.on('data', function(chunk) { + return resp.write(chunk); + }); + case 301: + if (remaining_redirects <= 0) { + return four_oh_four(resp, "Exceeded max depth"); + } + is_finished = false; + var url = Url.parse(srcResp.headers['location']); + return process_url(url, transferred_headers, resp, remaining_redirects - 1); + case 304: + return resp.writeHead(srcResp.statusCode, newHeaders); + default: + return four_oh_four(resp, "Responded with " + srcResp.statusCode + ":" + srcResp.headers); + } + } + }); + srcReq.on('error', function() { + return finish(resp); + }); + return srcReq.end(); + } else { + return four_oh_four(resp, "No host found " + url.host); + } + }; server = Http.createServer(function(req, resp) { var dest_url, encoded_url, hmac, hmac_digest, query_digest, query_path, src, srcReq, transferred_headers, url, url_type, _base, _ref; if (req.method !== 'GET' || req.url === '/') { @@ -89,71 +167,7 @@ hmac_digest = hmac.digest('hex'); if (hmac_digest === query_digest) { url = Url.parse(dest_url); - if ((url.host != null) && !url.host.match(RESTRICTED_IPS)) { - if (url.host.match(EXCLUDED_HOSTS)) { - return four_oh_four(resp, "Hitting excluded hostnames"); - } - src = Http.createClient(url.port || 80, url.hostname); - src.on('error', function(error) { - return four_oh_four(resp, "Client Request error " + error.stack); - }); - query_path = url.pathname; - if (url.query != null) { - query_path += "?" + url.query; - } - transferred_headers.host = url.host; - log(transferred_headers); - srcReq = src.request('GET', query_path, transferred_headers); - srcReq.on('response', function(srcResp) { - var content_length, newHeaders; - log(srcResp.headers); - content_length = srcResp.headers['content-length']; - if (content_length > 5242880) { - return four_oh_four(resp, "Content-Length exceeded"); - } else { - newHeaders = { - 'expires': srcResp.headers['expires'], - 'content-type': srcResp.headers['content-type'], - 'cache-control': srcResp.headers['cache-control'], - 'content-length': content_length, - 'Camo-Host': camo_hostname, - 'X-Content-Type-Options': 'nosniff' - }; - - if (srcResp.headers['content-encoding']) { - newHeaders['content-encoding'] = srcResp.headers['content-encoding']; - } - - srcResp.on('end', function() { - return finish(resp); - }); - srcResp.on('error', function() { - return finish(resp); - }); - switch (srcResp.statusCode) { - case 200: - if (newHeaders['content-type'] && newHeaders['content-type'].slice(0, 5) !== 'image') { - four_oh_four(resp, "Non-Image content-type returned"); - } - log(newHeaders); - resp.writeHead(srcResp.statusCode, newHeaders); - return srcResp.on('data', function(chunk) { - return resp.write(chunk); - }); - case 304: - return resp.writeHead(srcResp.statusCode, newHeaders); - default: - return four_oh_four(resp, "Responded with " + srcResp.statusCode + ":" + srcResp.headers); - } - } - }); - srcReq.on('error', function() { - return finish(resp); - }); - return srcReq.end(); - } else { - return four_oh_four(resp, "No host found " + url.host); - } + return process_url(url, transferred_headers, resp, max_redirects); } else { return four_oh_four(resp, "checksum mismatch " + hmac_digest + ":" + query_digest); } From 12b06d3172aa8f8a76e4652f277817aca08fc5b5 Mon Sep 17 00:00:00 2001 From: Dale Hui Date: Tue, 10 Jan 2012 12:56:06 -0800 Subject: [PATCH 04/15] Ported HTTP 301 support to the coffee version and regenerated server.js --- server.coffee | 144 ++++++++++++++++++++++++++------------------------ server.js | 87 ++++++++++++++++++------------ 2 files changed, 130 insertions(+), 101 deletions(-) diff --git a/server.coffee b/server.coffee index 66c41b3..698de93 100644 --- a/server.coffee +++ b/server.coffee @@ -10,6 +10,7 @@ excluded = process.env.CAMO_HOST_EXCLUSIONS || '*.example.org' shared_key = process.env.CAMO_KEY || '0x24FEEDFACEDEADBEEFCAFE' camo_hostname = process.env.CAMO_HOSTNAME || "unknown" logging_enabled = process.env.CAMO_LOGGING_ENABLED || "disabled" +max_redirects = process.env.CAMO_MAX_REDIRECTS || 10; log = (msg) -> unless logging_enabled == "disabled" @@ -34,6 +35,80 @@ finish = (resp, str) -> current_connections = 0 if current_connections < 1 resp.connection && resp.end str +process_url = (url, transferred_headers, resp, remaining_redirects) -> + if url.host? && !url.host.match(RESTRICTED_IPS) + if url.host.match(EXCLUDED_HOSTS) + return four_oh_four(resp, "Hitting excluded hostnames") + + src = Http.createClient url.port || 80, url.hostname + + src.on 'error', (error) -> + four_oh_four(resp, "Client Request error #{error.stack}") + + query_path = url.pathname + if url.query? + query_path += "?#{url.query}" + + transferred_headers.host = url.host + + log transferred_headers + + srcReq = src.request 'GET', query_path, transferred_headers + + srcReq.on 'response', (srcResp) -> + is_finished = true; + + log srcResp.headers + + content_length = srcResp.headers['content-length'] + + if content_length > 5242880 + four_oh_four(resp, "Content-Length exceeded"); + else + newHeaders = + 'expires' : srcResp.headers['expires'] + 'content-type' : srcResp.headers['content-type'] + 'cache-control' : srcResp.headers['cache-control'] + 'content-length' : content_length + 'Camo-Host' : camo_hostname + 'X-Content-Type-Options' : 'nosniff' + + if srcResp.headers['content-encoding'] + newHeaders['content-encoding'] = srcResp.headers['content-encoding'] + + srcResp.on 'end', -> + if is_finished + finish resp + srcResp.on 'error', -> + if is_finished + finish resp + switch srcResp.statusCode + when 200 + if newHeaders['content-type'] && newHeaders['content-type'].slice(0, 5) != 'image' + four_oh_four(resp, "Non-Image content-type returned") + + log newHeaders + + resp.writeHead srcResp.statusCode, newHeaders + srcResp.on 'data', (chunk) -> + resp.write chunk + when 301 + if remaining_redirects <= 0 + four_oh_four(resp, "Exceeded max depth") + is_finished = false; + url = Url.parse srcResp.headers['location'] + process_url url, transferred_headers, resp, remaining_redirects - 1 + when 304 + resp.writeHead srcResp.statusCode, newHeaders + else + four_oh_four(resp, "Responded with " + srcResp.statusCode + ":" + srcResp.headers) + srcReq.on 'error', -> + finish resp + + srcReq.end(); + else + four_oh_four(resp, "No host found " + url.host) + # decode a string of two char hex digits hexdec = (str) -> if str and str.length > 0 and str.length % 2 == 0 and not str.match(/[^0-9a-f]/) @@ -91,74 +166,7 @@ server = Http.createServer (req, resp) -> if hmac_digest == query_digest url = Url.parse dest_url - if url.host? && !url.host.match(RESTRICTED_IPS) - if url.host.match(EXCLUDED_HOSTS) - return four_oh_four(resp, "Hitting excluded hostnames") - - src = Http.createClient url.port || 80, url.hostname - - src.on 'error', (error) -> - four_oh_four(resp, "Client Request error #{error.stack}") - - query_path = url.pathname - if url.query? - query_path += "?#{url.query}" - - transferred_headers.host = url.host - - log transferred_headers - - srcReq = src.request 'GET', query_path, transferred_headers - - srcReq.on 'response', (srcResp) -> - log srcResp.headers - - content_length = srcResp.headers['content-length'] - - if content_length > 5242880 - four_oh_four(resp, "Content-Length exceeded") - else - newHeaders = - 'expires' : srcResp.headers['expires'] - 'content-type' : srcResp.headers['content-type'] - 'cache-control' : srcResp.headers['cache-control'] - 'content-length' : content_length - 'Camo-Host' : camo_hostname - 'X-Content-Type-Options' : 'nosniff' - - if srcResp.headers['content-encoding'] - newHeaders['content-encoding'] = srcResp.headers['content-encoding'] - - srcResp.on 'end', -> - finish resp - - srcResp.on 'error', -> - finish resp - - switch srcResp.statusCode - when 200 - if newHeaders['content-type'] && newHeaders['content-type'].slice(0, 5) != 'image' - four_oh_four(resp, "Non-Image content-type returned") - - log newHeaders - - resp.writeHead srcResp.statusCode, newHeaders - srcResp.on 'data', (chunk) -> - resp.write chunk - - when 304 - resp.writeHead srcResp.statusCode, newHeaders - - else - four_oh_four(resp, "Responded with #{srcResp.statusCode}:#{srcResp.headers}") - - srcReq.on 'error', -> - finish resp - - srcReq.end() - - else - four_oh_four(resp, "No host found #{url.host}") + process_url url, transferred_headers, resp, max_redirects else four_oh_four(resp, "checksum mismatch #{hmac_digest}:#{query_digest}") else diff --git a/server.js b/server.js index eddd833..c98f960 100644 --- a/server.js +++ b/server.js @@ -1,17 +1,30 @@ (function() { - var Crypto, EXCLUDED_HOSTS, Fs, Http, QueryString, RESTRICTED_IPS, Url, camo_hostname, current_connections, excluded, finish, four_oh_four, hexdec, log, logging_enabled, port, server, shared_key, started_at, total_connections, version; + var Crypto, EXCLUDED_HOSTS, Fs, Http, QueryString, RESTRICTED_IPS, Url, camo_hostname, current_connections, excluded, finish, four_oh_four, hexdec, log, logging_enabled, max_redirects, port, process_url, server, shared_key, started_at, total_connections, version; + Fs = require('fs'); + Url = require('url'); + Http = require('http'); + Crypto = require('crypto'); + QueryString = require('querystring'); + port = parseInt(process.env.PORT || 8081); + version = "0.3.0"; + excluded = process.env.CAMO_HOST_EXCLUSIONS || '*.example.org'; + shared_key = process.env.CAMO_KEY || '0x24FEEDFACEDEADBEEFCAFE'; + camo_hostname = process.env.CAMO_HOSTNAME || "unknown"; + logging_enabled = process.env.CAMO_LOGGING_ENABLED || "disabled"; + max_redirects = process.env.CAMO_MAX_REDIRECTS || 10; + log = function(msg) { if (logging_enabled !== "disabled") { console.log("--------------------------------------------"); @@ -19,52 +32,47 @@ return console.log("--------------------------------------------"); } }; + EXCLUDED_HOSTS = new RegExp(excluded.replace(".", "\\.").replace("*", "\\.*")); + RESTRICTED_IPS = /^((10\.)|(127\.)|(169\.254)|(192\.168)|(172\.((1[6-9])|(2[0-9])|(3[0-1]))))/; + total_connections = 0; + current_connections = 0; + started_at = new Date; + four_oh_four = function(resp, msg) { log(msg); resp.writeHead(404); return finish(resp, "Not Found"); }; + finish = function(resp, str) { current_connections -= 1; - if (current_connections < 1) { - current_connections = 0; - } + if (current_connections < 1) current_connections = 0; return resp.connection && resp.end(str); }; - hexdec = function(str) { - var buf, i, _ref, _step; - if (str && str.length > 0 && str.length % 2 === 0 && !str.match(/[^0-9a-f]/)) { - buf = new Buffer(str.length / 2); - for (i = 0, _ref = str.length, _step = 2; 0 <= _ref ? i < _ref : i > _ref; i += _step) { - buf[i / 2] = parseInt(str.slice(i, (i + 1 + 1) || 9e9), 16); - } - return buf.toString(); - } - }; + process_url = function(url, transferred_headers, resp, remaining_redirects) { + var query_path, src, srcReq; if ((url.host != null) && !url.host.match(RESTRICTED_IPS)) { if (url.host.match(EXCLUDED_HOSTS)) { return four_oh_four(resp, "Hitting excluded hostnames"); } - var src = Http.createClient(url.port || 80, url.hostname); + src = Http.createClient(url.port || 80, url.hostname); src.on('error', function(error) { return four_oh_four(resp, "Client Request error " + error.stack); }); - var query_path = url.pathname; - if (url.query != null) { - query_path += "?" + url.query; - } + query_path = url.pathname; + if (url.query != null) query_path += "?" + url.query; transferred_headers.host = url.host; log(transferred_headers); - var srcReq = src.request('GET', query_path, transferred_headers); + srcReq = src.request('GET', query_path, transferred_headers); srcReq.on('response', function(srcResp) { - var content_length, newHeaders; - var is_finished = true; + var content_length, is_finished, newHeaders; + is_finished = true; log(srcResp.headers); content_length = srcResp.headers['content-length']; if (content_length > 5242880) { @@ -78,23 +86,19 @@ 'Camo-Host': camo_hostname, 'X-Content-Type-Options': 'nosniff' }; - if (srcResp.headers['content-encoding']) { + if (srcResp.headers['content-encoding']) { newHeaders['content-encoding'] = srcResp.headers['content-encoding']; } srcResp.on('end', function() { - if (is_finished) { - return finish(resp); - } + if (is_finished) return finish(resp); }); srcResp.on('error', function() { - if (is_finished) { - return finish(resp); - } + if (is_finished) return finish(resp); }); switch (srcResp.statusCode) { case 200: if (newHeaders['content-type'] && newHeaders['content-type'].slice(0, 5) !== 'image') { - return four_oh_four(resp, "Non-Image content-type returned"); + four_oh_four(resp, "Non-Image content-type returned"); } log(newHeaders); resp.writeHead(srcResp.statusCode, newHeaders); @@ -103,10 +107,10 @@ }); case 301: if (remaining_redirects <= 0) { - return four_oh_four(resp, "Exceeded max depth"); + four_oh_four(resp, "Exceeded max depth"); } is_finished = false; - var url = Url.parse(srcResp.headers['location']); + url = Url.parse(srcResp.headers['location']); return process_url(url, transferred_headers, resp, remaining_redirects - 1); case 304: return resp.writeHead(srcResp.statusCode, newHeaders); @@ -123,8 +127,20 @@ return four_oh_four(resp, "No host found " + url.host); } }; + + hexdec = function(str) { + var buf, i, _ref; + if (str && str.length > 0 && str.length % 2 === 0 && !str.match(/[^0-9a-f]/)) { + buf = new Buffer(str.length / 2); + for (i = 0, _ref = str.length; i < _ref; i += 2) { + buf[i / 2] = parseInt(str.slice(i, (i + 1) + 1 || 9e9), 16); + } + return buf.toString(); + } + }; + server = Http.createServer(function(req, resp) { - var dest_url, encoded_url, hmac, hmac_digest, query_digest, query_path, src, srcReq, transferred_headers, url, url_type, _base, _ref; + var dest_url, encoded_url, hmac, hmac_digest, query_digest, transferred_headers, url, url_type, _base, _ref; if (req.method !== 'GET' || req.url === '/') { resp.writeHead(200); return resp.end('hwhat'); @@ -176,10 +192,15 @@ } } }); + console.log("SSL-Proxy running on " + port + " with pid:" + process.pid + "."); + console.log("Using the secret key " + shared_key); + Fs.open("tmp/camo.pid", "w", 0600, function(err, fd) { return Fs.writeSync(fd, process.pid); }); + server.listen(port); + }).call(this); From e6a9557f0e87b6c76b9b0f334c93095f68ac6674 Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Fri, 13 Jan 2012 15:24:49 -0800 Subject: [PATCH 05/15] align all the things --- server.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.coffee b/server.coffee index 698de93..cf0bd63 100644 --- a/server.coffee +++ b/server.coffee @@ -8,9 +8,9 @@ port = parseInt process.env.PORT || 8081 version = "0.3.0" excluded = process.env.CAMO_HOST_EXCLUSIONS || '*.example.org' shared_key = process.env.CAMO_KEY || '0x24FEEDFACEDEADBEEFCAFE' +max_redirects = process.env.CAMO_MAX_REDIRECTS || 4 camo_hostname = process.env.CAMO_HOSTNAME || "unknown" logging_enabled = process.env.CAMO_LOGGING_ENABLED || "disabled" -max_redirects = process.env.CAMO_MAX_REDIRECTS || 10; log = (msg) -> unless logging_enabled == "disabled" From 410eee1a3b10aa45c4cad5846a983910bdf54f04 Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Fri, 13 Jan 2012 19:30:32 -0800 Subject: [PATCH 06/15] remove ';' usage --- server.coffee | 8 ++++---- server.js | 1 + test/proxy_test.rb | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/server.coffee b/server.coffee index cf0bd63..47a1132 100644 --- a/server.coffee +++ b/server.coffee @@ -5,7 +5,7 @@ Crypto = require 'crypto' QueryString = require 'querystring' port = parseInt process.env.PORT || 8081 -version = "0.3.0" +version = "0.3.1" excluded = process.env.CAMO_HOST_EXCLUSIONS || '*.example.org' shared_key = process.env.CAMO_KEY || '0x24FEEDFACEDEADBEEFCAFE' max_redirects = process.env.CAMO_MAX_REDIRECTS || 4 @@ -56,14 +56,14 @@ process_url = (url, transferred_headers, resp, remaining_redirects) -> srcReq = src.request 'GET', query_path, transferred_headers srcReq.on 'response', (srcResp) -> - is_finished = true; + is_finished = true log srcResp.headers content_length = srcResp.headers['content-length'] if content_length > 5242880 - four_oh_four(resp, "Content-Length exceeded"); + four_oh_four(resp, "Content-Length exceeded") else newHeaders = 'expires' : srcResp.headers['expires'] @@ -105,7 +105,7 @@ process_url = (url, transferred_headers, resp, remaining_redirects) -> srcReq.on 'error', -> finish resp - srcReq.end(); + srcReq.end() else four_oh_four(resp, "No host found " + url.host) diff --git a/server.js b/server.js index c98f960..b4b7a74 100644 --- a/server.js +++ b/server.js @@ -106,6 +106,7 @@ return resp.write(chunk); }); case 301: + case 302: if (remaining_redirects <= 0) { four_oh_four(resp, "Exceeded max depth"); } diff --git a/test/proxy_test.rb b/test/proxy_test.rb index 20cef00..823e4ee 100644 --- a/test/proxy_test.rb +++ b/test/proxy_test.rb @@ -28,6 +28,11 @@ def test_proxy_valid_google_chart_url assert_equal(200, response.code) end + def test_follows_redirects + response = request('http://cl.ly/1K0X2Y2F1P0o3z140p0d/boom-headshot.gif') + assert_equal(200, response.code) + end + def test_404s_on_urls_without_an_http_host assert_raise RestClient::ResourceNotFound do request('/picture/Mincemeat/Pimp.jpg') From 11b14287db64125513a98cabe87221b212aafebf Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Fri, 13 Jan 2012 19:33:12 -0800 Subject: [PATCH 07/15] follow 302s, too --- server.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.coffee b/server.coffee index 47a1132..7835820 100644 --- a/server.coffee +++ b/server.coffee @@ -92,10 +92,10 @@ process_url = (url, transferred_headers, resp, remaining_redirects) -> resp.writeHead srcResp.statusCode, newHeaders srcResp.on 'data', (chunk) -> resp.write chunk - when 301 + when 301, 302 if remaining_redirects <= 0 four_oh_four(resp, "Exceeded max depth") - is_finished = false; + is_finished = false url = Url.parse srcResp.headers['location'] process_url url, transferred_headers, resp, remaining_redirects - 1 when 304 From 8e497d09ae55cc21ae892869bf691847ce2fcade Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Fri, 13 Jan 2012 19:58:16 -0800 Subject: [PATCH 08/15] test hella redirect failures --- test/proxy_test.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/proxy_test.rb b/test/proxy_test.rb index 823e4ee..36d0472 100644 --- a/test/proxy_test.rb +++ b/test/proxy_test.rb @@ -33,6 +33,12 @@ def test_follows_redirects assert_equal(200, response.code) end + def test_404s_on_infinidirect + assert_raise RestClient::ResourceNotFound do + request('http://modeselektor.herokuapp.com/') + end + end + def test_404s_on_urls_without_an_http_host assert_raise RestClient::ResourceNotFound do request('/picture/Mincemeat/Pimp.jpg') From c9b3c31723054f536db61402741f3de96382418c Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Fri, 13 Jan 2012 23:47:15 -0800 Subject: [PATCH 09/15] start a changelog --- CHANGELOG.md | 5 +++++ README.md | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c5020be --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +0.3.1 +===== + +* Follow redirects to a configurable depth + diff --git a/README.md b/README.md index ae54999..c2aa44b 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ Features * Proxy remote images with a content-type of `image/*` * Proxy images under 5 MB * Proxy google charts -* 404s for anything other than a 200 or 304 HTTP response +* Follow redirects to a configurable depth +* 404s for anything other than a 200, 301, 302 or 304 HTTP response * Disallows proxying to private IP ranges At GitHub we render markdown and replace all of the `src` attributes on the `img` tags with the appropriate URL to hit the proxies. There's example code for creating URLs in [the tests](https://github.com/atmos/camo/blob/master/test/proxy_test.rb). From b20f4efe8e263315f0e65bd7a3e2a25b1104b9c3 Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Fri, 13 Jan 2012 23:55:38 -0800 Subject: [PATCH 10/15] version bumps --- CHANGELOG.md | 2 +- README.md | 6 +++--- server.coffee | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5020be..91fdd16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -0.3.1 +0.5.0 ===== * Follow redirects to a configurable depth diff --git a/README.md b/README.md index c2aa44b..73d8f5c 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,15 @@ We want to allow people to keep embedding images in comments/issues/READMEs/goog Using a shared key, proxy URLs are encrypted with [hmac](http://en.wikipedia.org/wiki/HMAC) so we can bust caches/ban/rate limit if needed. -Camo currently runs on node version 0.4.8 in production at GitHub. +Camo currently runs on node version 0.4.10 at GitHub. Features -------- -* Proxy remote images with a content-type of `image/*` -* Proxy images under 5 MB * Proxy google charts +* Proxy images under 5 MB * Follow redirects to a configurable depth +* Proxy remote images with a content-type of `image/*` * 404s for anything other than a 200, 301, 302 or 304 HTTP response * Disallows proxying to private IP ranges diff --git a/server.coffee b/server.coffee index 7835820..2460445 100644 --- a/server.coffee +++ b/server.coffee @@ -5,7 +5,7 @@ Crypto = require 'crypto' QueryString = require 'querystring' port = parseInt process.env.PORT || 8081 -version = "0.3.1" +version = "0.5.0" excluded = process.env.CAMO_HOST_EXCLUSIONS || '*.example.org' shared_key = process.env.CAMO_KEY || '0x24FEEDFACEDEADBEEFCAFE' max_redirects = process.env.CAMO_MAX_REDIRECTS || 4 From b3a0843915e7dafa42fdc036236b315846000e6a Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Sat, 14 Jan 2012 22:29:42 -0800 Subject: [PATCH 11/15] fixup indenting an generated server.js --- server.coffee | 12 ++++++------ server.js | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server.coffee b/server.coffee index 2460445..ed3ab6d 100644 --- a/server.coffee +++ b/server.coffee @@ -66,12 +66,12 @@ process_url = (url, transferred_headers, resp, remaining_redirects) -> four_oh_four(resp, "Content-Length exceeded") else newHeaders = - 'expires' : srcResp.headers['expires'] - 'content-type' : srcResp.headers['content-type'] - 'cache-control' : srcResp.headers['cache-control'] - 'content-length' : content_length - 'Camo-Host' : camo_hostname - 'X-Content-Type-Options' : 'nosniff' + 'expires' : srcResp.headers['expires'] + 'content-type' : srcResp.headers['content-type'] + 'cache-control' : srcResp.headers['cache-control'] + 'content-length' : content_length + 'Camo-Host' : camo_hostname + 'X-Content-Type-Options' : 'nosniff' if srcResp.headers['content-encoding'] newHeaders['content-encoding'] = srcResp.headers['content-encoding'] diff --git a/server.js b/server.js index b4b7a74..42f4efa 100644 --- a/server.js +++ b/server.js @@ -13,18 +13,18 @@ port = parseInt(process.env.PORT || 8081); - version = "0.3.0"; + version = "0.5.0"; excluded = process.env.CAMO_HOST_EXCLUSIONS || '*.example.org'; shared_key = process.env.CAMO_KEY || '0x24FEEDFACEDEADBEEFCAFE'; + max_redirects = process.env.CAMO_MAX_REDIRECTS || 4; + camo_hostname = process.env.CAMO_HOSTNAME || "unknown"; logging_enabled = process.env.CAMO_LOGGING_ENABLED || "disabled"; - max_redirects = process.env.CAMO_MAX_REDIRECTS || 10; - log = function(msg) { if (logging_enabled !== "disabled") { console.log("--------------------------------------------"); From 7e756f77582d66c7321d0eb44c1a6ac72bc5ed7e Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Tue, 17 Jan 2012 16:24:31 -0800 Subject: [PATCH 12/15] fix a redirect follow bug with location headers Redirection without a hostname in the location headers would 404 --- server.coffee | 8 ++++++-- test/proxy_test.rb | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/server.coffee b/server.coffee index ed3ab6d..99d38cf 100644 --- a/server.coffee +++ b/server.coffee @@ -96,8 +96,12 @@ process_url = (url, transferred_headers, resp, remaining_redirects) -> if remaining_redirects <= 0 four_oh_four(resp, "Exceeded max depth") is_finished = false - url = Url.parse srcResp.headers['location'] - process_url url, transferred_headers, resp, remaining_redirects - 1 + newUrl = Url.parse srcResp.headers['location'] + unless newUrl.host? and newUrl.hostname? + newUrl.host = newUrl.hostname = url.hostname + newUrl.protocol = url.protocol + + process_url newUrl, transferred_headers, resp, remaining_redirects - 1 when 304 resp.writeHead srcResp.statusCode, newHeaders else diff --git a/test/proxy_test.rb b/test/proxy_test.rb index 36d0472..4ee07ec 100644 --- a/test/proxy_test.rb +++ b/test/proxy_test.rb @@ -4,6 +4,7 @@ require 'openssl' require 'rest_client' require 'addressable/uri' +require 'ruby-debug' require 'test/unit' @@ -33,6 +34,17 @@ def test_follows_redirects assert_equal(200, response.code) end + def test_follows_redirects_formatted_strangely + response = request('http://cl.ly/DPcp/Screen%20Shot%202012-01-17%20at%203.42.32%20PM.png') + assert_equal(200, response.code) + end + + def test_follows_redirects_with_path_only_location_headers + assert_nothing_raised do + request('http://blogs.msdn.com/photos/noahric/images/9948044/425x286.aspx') + end + end + def test_404s_on_infinidirect assert_raise RestClient::ResourceNotFound do request('http://modeselektor.herokuapp.com/') @@ -51,12 +63,6 @@ def test_404s_on_images_greater_than_5_megabytes end end - def test_404s_on_redirects - assert_raise RestClient::ResourceNotFound do - request('http://blogs.msdn.com/photos/noahric/images/9948044/425x286.aspx') - end - end - def test_404s_on_host_not_found assert_raise RestClient::ResourceNotFound do request('http://flabergasted.cx') From 05e0e9f89e2956608c0033e7d06451022abc3863 Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Tue, 17 Jan 2012 16:25:25 -0800 Subject: [PATCH 13/15] latest server.js --- server.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server.js b/server.js index 42f4efa..77952d9 100644 --- a/server.js +++ b/server.js @@ -71,7 +71,7 @@ log(transferred_headers); srcReq = src.request('GET', query_path, transferred_headers); srcReq.on('response', function(srcResp) { - var content_length, is_finished, newHeaders; + var content_length, is_finished, newHeaders, newUrl; is_finished = true; log(srcResp.headers); content_length = srcResp.headers['content-length']; @@ -111,8 +111,12 @@ four_oh_four(resp, "Exceeded max depth"); } is_finished = false; - url = Url.parse(srcResp.headers['location']); - return process_url(url, transferred_headers, resp, remaining_redirects - 1); + newUrl = Url.parse(srcResp.headers['location']); + if (!((newUrl.host != null) && (newUrl.hostname != null))) { + newUrl.host = newUrl.hostname = url.hostname; + newUrl.protocol = url.protocol; + } + return process_url(newUrl, transferred_headers, resp, remaining_redirects - 1); case 304: return resp.writeHead(srcResp.statusCode, newHeaders); default: From b3e1af78e7cb5befa4efecf7d23aff6779812812 Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Tue, 17 Jan 2012 18:19:10 -0800 Subject: [PATCH 14/15] actually stop following after N redirects --- server.coffee | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/server.coffee b/server.coffee index 99d38cf..d7b3e77 100644 --- a/server.coffee +++ b/server.coffee @@ -95,13 +95,15 @@ process_url = (url, transferred_headers, resp, remaining_redirects) -> when 301, 302 if remaining_redirects <= 0 four_oh_four(resp, "Exceeded max depth") - is_finished = false - newUrl = Url.parse srcResp.headers['location'] - unless newUrl.host? and newUrl.hostname? - newUrl.host = newUrl.hostname = url.hostname - newUrl.protocol = url.protocol - - process_url newUrl, transferred_headers, resp, remaining_redirects - 1 + else + is_finished = false + newUrl = Url.parse srcResp.headers['location'] + unless newUrl.host? and newUrl.hostname? + newUrl.host = newUrl.hostname = url.hostname + newUrl.protocol = url.protocol + + console.log newUrl + process_url newUrl, transferred_headers, resp, remaining_redirects - 1 when 304 resp.writeHead srcResp.statusCode, newHeaders else From 4d3bf19c9e35319916abd6f46eea8aaea18531e1 Mon Sep 17 00:00:00 2001 From: Corey Donohoe Date: Tue, 17 Jan 2012 18:22:50 -0800 Subject: [PATCH 15/15] re-render server.js --- server.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/server.js b/server.js index 77952d9..68c68fd 100644 --- a/server.js +++ b/server.js @@ -108,15 +108,18 @@ case 301: case 302: if (remaining_redirects <= 0) { - four_oh_four(resp, "Exceeded max depth"); + return four_oh_four(resp, "Exceeded max depth"); + } else { + is_finished = false; + newUrl = Url.parse(srcResp.headers['location']); + if (!((newUrl.host != null) && (newUrl.hostname != null))) { + newUrl.host = newUrl.hostname = url.hostname; + newUrl.protocol = url.protocol; + } + console.log(newUrl); + return process_url(newUrl, transferred_headers, resp, remaining_redirects - 1); } - is_finished = false; - newUrl = Url.parse(srcResp.headers['location']); - if (!((newUrl.host != null) && (newUrl.hostname != null))) { - newUrl.host = newUrl.hostname = url.hostname; - newUrl.protocol = url.protocol; - } - return process_url(newUrl, transferred_headers, resp, remaining_redirects - 1); + break; case 304: return resp.writeHead(srcResp.statusCode, newHeaders); default: