diff --git a/README.md b/README.md index 21cd9de..ae54999 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,33 @@ We want to allow people to keep embedding images in comments/issues/READMEs/goog Using a shared key, proxy URLs are encrypted with [hmac](http://en.wikipedia.org/wiki/HMAC) so we can bust caches/ban/rate limit if needed. +Camo currently runs on node version 0.4.8 in production at GitHub. + Features -------- -* Proxy remote images with a content-type of images/* -* Proxy images < 5 MB +* Proxy remote images with a content-type of `image/*` +* Proxy images under 5 MB * Proxy google charts * 404s for anything other than a 200 or 304 HTTP response * Disallows proxying to private IP ranges At GitHub we render markdown and replace all of the `src` attributes on the `img` tags with the appropriate URL to hit the proxies. There's example code for creating URLs in [the tests](https://github.com/atmos/camo/blob/master/test/proxy_test.rb). +## URL Formats + +Camo supports two distinct URL formats: + + http://example.org/?url= + http://example.org// + +The `` is a 40 character hex encoded HMAC digest generated with a shared +secret key and the unescaped `` value. The `` is the +absolute URL locating an image. In the first format, the `` should be +URL escaped aggressively to ensure the original value isn't mangled in transit. +In the second format, each byte of the `` should be hex encoded such +that the resulting value includes only characters `[0-9a-f]`. + ## Testing Functionality ### Start the server diff --git a/Rakefile b/Rakefile index f236fee..47f4242 100644 --- a/Rakefile +++ b/Rakefile @@ -1,10 +1,15 @@ +file 'server.js' => 'server.coffee' do + sh "coffee -c -o . server.coffee" +end +task :build => 'server.js' + namespace :test do desc "Run the tests against localhost" task :check do |t| system("ruby test/proxy_test.rb") end end -task :default => "test:check" +task :default => [:build, "test:check"] Dir["tasks/*.rake"].each do |f| load f diff --git a/server.coffee b/server.coffee index aa63f98..4379ab7 100644 --- a/server.coffee +++ b/server.coffee @@ -8,6 +8,7 @@ port = process.env.PORT || 8081 version = "0.3.0" excluded = process.env.CAMO_HOST_EXCLUSIONS || '*.example.org' shared_key = process.env.CAMO_KEY || '0x24FEEDFACEDEADBEEFCAFE' +camo_hostname = process.env.CAMO_HOSTNAME || "unknown" logging_enabled = process.env.CAMO_LOGGING_ENABLED || "disabled" pidfile = process.env.PIDFILE || 'tmp/camo.pid' @@ -18,7 +19,7 @@ log = (msg) -> console.log("--------------------------------------------") EXCLUDED_HOSTS = new RegExp(excluded.replace(".", "\\.").replace("*", "\\.*")) -RESTRICTED_IPS = /^(10\.)|(127\.)|(169\.254)|(192\.168)|(172\.(1[6-9])|(2[0-9])|(3[0-1]))/ +RESTRICTED_IPS = /^((10\.)|(127\.)|(169\.254)|(192\.168)|(172\.((1[6-9])|(2[0-9])|(3[0-1]))))/ total_connections = 0 current_connections = 0 @@ -32,7 +33,15 @@ four_oh_four = (resp, msg) -> finish = (resp, str) -> current_connections -= 1 current_connections = 0 if current_connections < 1 - resp.end str + resp.connection && resp.end str + +# decode a string of two char hex digits +hexdec = (str) -> + if str and str.length > 0 and str.length % 2 == 0 and not str.match(/[^0-9a-f]/) + buf = new Buffer(str.length / 2) + for i in [0...str.length] by 2 + buf[i/2] = parseInt(str[i..i+1], 16) + buf.toString() server = Http.createServer (req, resp) -> if req.method != 'GET' || req.url == '/' @@ -57,18 +66,31 @@ server = Http.createServer (req, resp) -> 'x-content-type-options' : 'nosniff' delete(req.headers.cookie) - log(req.headers) - - query_digest = url.pathname.replace(/^\//, '') - query_params = QueryString.parse(url.query) - if url.pathname? + [query_digest, encoded_url] = url.pathname.replace(/^\//, '').split("/", 2) + if encoded_url = hexdec(encoded_url) + url_type = 'path' + dest_url = encoded_url + else + url_type = 'query' + dest_url = QueryString.parse(url.query).url + + log({ + type: url_type + url: req.url + headers: req.headers + dest: dest_url + digest: query_digest + }) + + if url.pathname? && dest_url hmac = Crypto.createHmac("sha1", shared_key) - hmac.update(query_params.url) + hmac.update(dest_url) + hmac_digest = hmac.digest('hex') if hmac_digest == query_digest - url = Url.parse query_params.url + url = Url.parse dest_url if url.host? && !url.host.match(RESTRICTED_IPS) if url.host.match(EXCLUDED_HOSTS) @@ -102,6 +124,7 @@ server = Http.createServer (req, resp) -> 'content-type' : srcResp.headers['content-type'] 'cache-control' : srcResp.headers['cache-control'] 'content-length' : content_length + 'Camo-Host' : camo_hostname 'X-Content-Type-Options' : 'nosniff' srcResp.on 'end', -> diff --git a/test/proxy_test.rb b/test/proxy_test.rb index 4c49308..20cef00 100644 --- a/test/proxy_test.rb +++ b/test/proxy_test.rb @@ -7,27 +7,22 @@ require 'test/unit' -class CamoProxyTest < Test::Unit::TestCase +module CamoProxyTests def config { 'key' => ENV['CAMO_KEY'] || "0x24FEEDFACEDEADBEEFCAFE", 'host' => ENV['CAMO_HOST'] || "http://localhost:8081" } end - def request(image_url) - hexdigest = OpenSSL::HMAC.hexdigest( - OpenSSL::Digest::Digest.new('sha1'), config['key'], image_url) - - uri = Addressable::URI.parse("#{config['host']}/#{hexdigest}") - uri.query_values = { 'url' => image_url, 'repo' => '', 'path' => '' } - - RestClient.get(uri.to_s) - end - def test_proxy_valid_image_url response = request('http://media.ebaumsworld.com/picture/Mincemeat/Pimp.jpg') assert_equal(200, response.code) end + def test_proxy_valid_image_url_with_crazy_subdomain + response = request('http://27.media.tumblr.com/tumblr_lkp6rdDfRi1qce6mto1_500.jpg') + assert_equal(200, response.code) + end + def test_proxy_valid_google_chart_url response = request('http://chart.apis.google.com/chart?chs=920x200&chxl=0:%7C2010-08-13%7C2010-09-12%7C2010-10-12%7C2010-11-11%7C1:%7C0%7C0%7C0%7C0%7C0%7C0&chm=B,EBF5FB,0,0,0&chco=008Cd6&chls=3,1,0&chg=8.3,20,1,4&chd=s:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&chxt=x,y&cht=lc') assert_equal(200, response.code) @@ -95,3 +90,40 @@ def test_404s_on_environmental_excludes end end end + +class CamoProxyQueryStringTest < Test::Unit::TestCase + include CamoProxyTests + + def request_uri(image_url) + hexdigest = OpenSSL::HMAC.hexdigest( + OpenSSL::Digest::Digest.new('sha1'), config['key'], image_url) + + uri = Addressable::URI.parse("#{config['host']}/#{hexdigest}") + uri.query_values = { 'url' => image_url, 'repo' => '', 'path' => '' } + + uri.to_s + end + + def request(image_url) + RestClient.get(request_uri(image_url)) + end +end + +class CamoProxyPathTest < Test::Unit::TestCase + include CamoProxyTests + + def hexenc(image_url) + image_url.to_enum(:each_byte).map { |byte| "%02x" % byte }.join + end + + def request_uri(image_url) + hexdigest = OpenSSL::HMAC.hexdigest( + OpenSSL::Digest::Digest.new('sha1'), config['key'], image_url) + encoded_image_url = hexenc(image_url) + "#{config['host']}/#{hexdigest}/#{encoded_image_url}" + end + + def request(image_url) + RestClient.get(request_uri(image_url)) + end +end