Merge pull request #9 from soundcloud/refactor-vendoring

Refactor vendoring

README.md

@@ -297,5 +297,15 @@ Go installed on your computer to build Roshi. If you're on a Mac and use
 
 # I just want to run the HTTP service!
 
-    go get github.com/soundcloud/roshi/roshi-server
-    roshi-server -h
+roshi-server uses vendored dependencies and a "blessed build" process to ensure
+stability. The recommended way to get and build roshi-server is
+
+    git clone [email protected]:soundcloud/roshi
+	cd roshi/roshi-server
+	make
+
+Invoking `go get` directly (`go get github.com/soundcloud/roshi/roshi-server`)
+should work, with the caveat that it will download missing dependencies
+directly from their respective master branches, which may not be properly
+tested.
+

cluster/cluster.go

@@ -13,7 +13,7 @@ import (
 	"github.com/soundcloud/roshi/common"
 	"github.com/soundcloud/roshi/instrumentation"
 	"github.com/soundcloud/roshi/shard"
-	"github.com/soundcloud/roshi/vendor/redigo/redis"
+	"github.com/garyburd/redigo/redis"
 )
 
 func init() {

cluster/cluster_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/soundcloud/roshi/cluster"
 	"github.com/soundcloud/roshi/common"
 	"github.com/soundcloud/roshi/shard"
-	"github.com/soundcloud/roshi/vendor/redigo/redis"
+	"github.com/garyburd/redigo/redis"
 )
 
 func TestInsertSelectKeys(t *testing.T) {

instrumentation/statsd/statsd.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/soundcloud/roshi/instrumentation"
-	"github.com/soundcloud/roshi/vendor/g2s"
+	"github.com/peterbourgon/g2s"
 )
 
 type statsdInstrumentation struct {

roshi-server/Makefile

@@ -0,0 +1,13 @@
+GOPATH := $(CURDIR)/../vendor:$(GOPATH)
+
+all: build
+
+build:
+	go build
+
+clean:
+	go clean
+
+check:
+	@go list -f '{{join .Deps "\n"}}' | xargs go list -f '{{if not .Standard}}{{.ImportPath}} {{.Dir}}{{end}}' | column -t
+

roshi-server/README.md

@@ -9,11 +9,16 @@ satisfy your load volume.
 
 [farm]: http://github.com/soundcloud/roshi/blob/master/farm
 
-## Getting
+## Getting and building
 
-```
-go get github.com/soundcloud/roshi/roshi-server
-```
+roshi-server uses vendored dependencies and a "blessed build" process to ensure
+stability over time. Users should get and build roshi-server by cloning this
+repository and running `make` in the roshi-server subdirectory. A working Go
+toolchain is assumed.
+
+It's also possible to build roshi-server with a simple `go build`, with the
+caveat that it will use your normal GOPATH to resolve dependencies, and
+therefore will enforce no constraints on dependency versions.
 
 ## Running
 
@@ -23,6 +28,10 @@ As a demo, start an instance of Redis on the standard port, and run
 roshi-server -redis.instances=localhost:6379
 ```
 
+This will create a farm with a single cluster containing a single Redis
+instance. All functionality will work as advertised, albeit with effectively
+zero fault-tolerance.
+
 ## API
 
 The server installs one handler on the root path. Operations are
@@ -131,3 +140,5 @@ roshi-server will use very little RAM and comparatively large amount of CPU.
 It may make sense to co-locate a roshi-server instance with every Redis
 instance. (It's been our experience that a single server-class machine is best
 utilized when it runs multiple Redis instances.)
+
+

roshi-server/main.go

@@ -23,9 +23,9 @@ import (
 	"github.com/soundcloud/roshi/farm"
 	"github.com/soundcloud/roshi/instrumentation/statsd"
 	"github.com/soundcloud/roshi/shard"
-	"github.com/soundcloud/roshi/vendor/g2s"
-	"github.com/soundcloud/roshi/vendor/handy/breaker"
-	"github.com/soundcloud/roshi/vendor/pat"
+	"github.com/peterbourgon/g2s"
+	"github.com/streadway/handy/breaker"
+	"github.com/gorilla/pat"
 )
 
 const (

roshi-server/main_test.go

@@ -12,7 +12,7 @@ import (
 	"testing"
 
 	"github.com/soundcloud/roshi/common"
-	"github.com/soundcloud/roshi/vendor/pat"
+	"github.com/gorilla/pat"
 )
 
 func TestEvaluateScalarPercentage(t *testing.T) {

shard/connection_pool.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/soundcloud/roshi/vendor/redigo/redis"
+	"github.com/garyburd/redigo/redis"
 )
 
 var errDialDeadline = errors.New("couldn't successfully dial an instance")

shard/integration_test.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/soundcloud/roshi/shard"
-	"github.com/soundcloud/roshi/vendor/redigo/redis"
+	"github.com/garyburd/redigo/redis"
 )
 
 func TestRecovery(t *testing.T) {

shard/shard.go

@@ -4,7 +4,7 @@ package shard
 import (
 	"time"
 
-	"github.com/soundcloud/roshi/vendor/redigo/redis"
+	"github.com/garyburd/redigo/redis"
 )
 
 // Shards maintains a connection pool for multiple Redis instances.

vendor/README.md

@@ -7,9 +7,9 @@ All management commands should be run from the top level of the repository.
 
 ## Adding a new dependency
 
-    git subtree add --prefix vendor/bar [email protected]:foo/bar master --squash
+    git subtree add --prefix vendor/src/github.com/foo/bar [email protected]:foo/bar master --squash
 
 ## Updating a dependency
 
-    git subtree pull --prefix vendor/bar [email protected]:foo/bar master --squash
+    git subtree pull --prefix vendor/src/github.com/foo/bar [email protected]:foo/bar master --squash
 

vendor/src/github.com/gorilla/context/.travis.yml

@@ -0,0 +1,7 @@
+language: go
+
+go:
+  - 1.0
+  - 1.1
+  - 1.2
+  - tip

vendor/src/github.com/gorilla/context/README.md

@@ -0,0 +1,7 @@
+context
+=======
+[![Build Status](https://travis-ci.org/gorilla/context.png?branch=master)](https://travis-ci.org/gorilla/context)
+
+gorilla/context is a general purpose registry for global request variables.
+
+Read the full documentation here: http://www.gorillatoolkit.org/pkg/context

vendor/src/github.com/gorilla/context/context.go

@@ -0,0 +1,112 @@
+// Copyright 2012 The Gorilla Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package context
+
+import (
+	"net/http"
+	"sync"
+	"time"
+)
+
+var (
+	mutex sync.Mutex
+	data  = make(map[*http.Request]map[interface{}]interface{})
+	datat = make(map[*http.Request]int64)
+)
+
+// Set stores a value for a given key in a given request.
+func Set(r *http.Request, key, val interface{}) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if data[r] == nil {
+		data[r] = make(map[interface{}]interface{})
+		datat[r] = time.Now().Unix()
+	}
+	data[r][key] = val
+}
+
+// Get returns a value stored for a given key in a given request.
+func Get(r *http.Request, key interface{}) interface{} {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if data[r] != nil {
+		return data[r][key]
+	}
+	return nil
+}
+
+// GetOk returns stored value and presence state like multi-value return of map access.
+func GetOk(r *http.Request, key interface{}) (interface{}, bool) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if _, ok := data[r]; ok {
+		value, ok := data[r][key]
+		return value, ok
+	}
+	return nil, false
+}
+
+// Delete removes a value stored for a given key in a given request.
+func Delete(r *http.Request, key interface{}) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if data[r] != nil {
+		delete(data[r], key)
+	}
+}
+
+// Clear removes all values stored for a given request.
+//
+// This is usually called by a handler wrapper to clean up request
+// variables at the end of a request lifetime. See ClearHandler().
+func Clear(r *http.Request) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	clear(r)
+}
+
+// clear is Clear without the lock.
+func clear(r *http.Request) {
+	delete(data, r)
+	delete(datat, r)
+}
+
+// Purge removes request data stored for longer than maxAge, in seconds.
+// It returns the amount of requests removed.
+//
+// If maxAge <= 0, all request data is removed.
+//
+// This is only used for sanity check: in case context cleaning was not
+// properly set some request data can be kept forever, consuming an increasing
+// amount of memory. In case this is detected, Purge() must be called
+// periodically until the problem is fixed.
+func Purge(maxAge int) int {
+	mutex.Lock()
+	defer mutex.Unlock()
+	count := 0
+	if maxAge <= 0 {
+		count = len(data)
+		data = make(map[*http.Request]map[interface{}]interface{})
+		datat = make(map[*http.Request]int64)
+	} else {
+		min := time.Now().Unix() - int64(maxAge)
+		for r := range data {
+			if datat[r] < min {
+				clear(r)
+				count++
+			}
+		}
+	}
+	return count
+}
+
+// ClearHandler wraps an http.Handler and clears request values at the end
+// of a request lifetime.
+func ClearHandler(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer Clear(r)
+		h.ServeHTTP(w, r)
+	})
+}

vendor/src/github.com/gorilla/context/context_test.go

@@ -0,0 +1,66 @@
+// Copyright 2012 The Gorilla Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package context
+
+import (
+	"net/http"
+	"testing"
+)
+
+type keyType int
+
+const (
+	key1 keyType = iota
+	key2
+)
+
+func TestContext(t *testing.T) {
+	assertEqual := func(val interface{}, exp interface{}) {
+		if val != exp {
+			t.Errorf("Expected %v, got %v.", exp, val)
+		}
+	}
+
+	r, _ := http.NewRequest("GET", "http://localhost:8080/", nil)
+
+	// Get()
+	assertEqual(Get(r, key1), nil)
+
+	// Set()
+	Set(r, key1, "1")
+	assertEqual(Get(r, key1), "1")
+	assertEqual(len(data[r]), 1)
+
+	Set(r, key2, "2")
+	assertEqual(Get(r, key2), "2")
+	assertEqual(len(data[r]), 2)
+
+	//GetOk
+	value, ok := GetOk(r, key1)
+	assertEqual(value, "1")
+	assertEqual(ok, true)
+
+	value, ok = GetOk(r, "not exists")
+	assertEqual(value, nil)
+	assertEqual(ok, false)
+
+	Set(r, "nil value", nil)
+	value, ok = GetOk(r, "nil value")
+	assertEqual(value, nil)
+	assertEqual(ok, true)
+
+	// Delete()
+	Delete(r, key1)
+	assertEqual(Get(r, key1), nil)
+	assertEqual(len(data[r]), 2)
+
+	Delete(r, key2)
+	assertEqual(Get(r, key2), nil)
+	assertEqual(len(data[r]), 1)
+
+	// Clear()
+	Clear(r)
+	assertEqual(len(data), 0)
+}