diff --git a/.gitignore b/.gitignore index aa2f43f..7c54386 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ db.gz *.csv *.zip *.sqlite +/vendor/*/ diff --git a/Dockerfile b/Dockerfile index ad6e594..90d3c9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,8 @@ RUN cd /go/src/github.com/fiorix/freegeoip/cmd/freegeoip && go get && go install ENTRYPOINT ["/go/bin/freegeoip"] +EXPOSE 8080 + # CMD instructions: # Add "-use-x-forwarded-for" if your server is behind a reverse proxy # Add "-public", "/var/www" to enable the web front-end diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..de810e1 --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: freegeoip -http :${PORT} -use-x-forwarded-for -public /app/cmd/freegeoip/public -quota-backend map -quota-max 10000 diff --git a/README.md b/README.md index 9c2f36e..0d27e0d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # freegeoip +[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) + This is the source code of the freegeoip software. It contains both the web server that empowers freegeoip.net, and a package for the [Go](http://golang.org) programming language that enables any web server diff --git a/apiserver/api.go b/apiserver/api.go index f7ae779..1296228 100644 --- a/apiserver/api.go +++ b/apiserver/api.go @@ -27,6 +27,7 @@ import ( "github.com/go-web/httprl" "github.com/go-web/httprl/memcacherl" "github.com/go-web/httprl/redisrl" + newrelic "github.com/newrelic/go-agent" "github.com/prometheus/client_golang/prometheus" "github.com/rs/cors" @@ -34,9 +35,10 @@ import ( ) type apiHandler struct { - db *freegeoip.DB - conf *Config - cors *cors.Cors + db *freegeoip.DB + conf *Config + cors *cors.Cors + nrapp newrelic.Application } // NewHandler creates an http handler for the freegeoip server that @@ -83,6 +85,14 @@ func (f *apiHandler) config(mc *httpmux.Config) error { } mc.Use(rl.Handle) } + if f.conf.NewrelicName != "" && f.conf.NewrelicKey != "" { + config := newrelic.NewConfig(f.conf.NewrelicName, f.conf.NewrelicKey) + app, err := newrelic.NewApplication(config) + if err != nil { + return fmt.Errorf("failed to create newrelic application: {name: %v, key: %v}", f.conf.NewrelicName, f.conf.NewrelicKey) + } + f.nrapp = app + } return nil } @@ -126,7 +136,13 @@ func (f *apiHandler) metrics(next http.HandlerFunc) http.HandlerFunc { type writerFunc func(w http.ResponseWriter, r *http.Request, d *responseRecord) func (f *apiHandler) register(name string, writer writerFunc) http.HandlerFunc { - h := prometheus.InstrumentHandler(name, f.iplookup(writer)) + var h http.Handler + if f.nrapp == nil { + h = prometheus.InstrumentHandler(name, f.iplookup(writer)) + } else { + h = prometheus.InstrumentHandler(newrelic.WrapHandle(f.nrapp, name, f.iplookup(writer))) + } + return f.cors.Handler(h).ServeHTTP } diff --git a/apiserver/config.go b/apiserver/config.go index adc9c5c..1105cfc 100644 --- a/apiserver/config.go +++ b/apiserver/config.go @@ -50,6 +50,8 @@ type Config struct { LicenseKey string UserID string ProductID string + NewrelicName string + NewrelicKey string errorLog *log.Logger accessLog *log.Logger @@ -122,6 +124,8 @@ func (c *Config) AddFlags(fs *flag.FlagSet) { fs.StringVar(&c.LicenseKey, "license-key", c.LicenseKey, "MaxMind License Key (requires user-id)") fs.StringVar(&c.UserID, "user-id", c.UserID, "MaxMind User ID (requires license-key)") fs.StringVar(&c.ProductID, "product-id", c.ProductID, "MaxMind Product ID (e.g GeoIP2-City)") + fs.StringVar(&c.NewrelicName, "newrelic-name", c.NewrelicName, "Newrepic APM application name") + fs.StringVar(&c.NewrelicKey, "newrelic-key", c.NewrelicKey, "Nerelic API key") } func (c *Config) logWriter() io.Writer { diff --git a/app.json b/app.json new file mode 100644 index 0000000..c9f0311 --- /dev/null +++ b/app.json @@ -0,0 +1,7 @@ +{ + "name": "freegeoip", + "description": "IP geolocation web server", + "website": "https://github.com/fiorix/freegeoip", + "success_url": "/", + "keywords": ["golang", "geoip", "api"] +} \ No newline at end of file diff --git a/db.go b/db.go index 1550dbb..2810a7e 100644 --- a/db.go +++ b/db.go @@ -41,6 +41,7 @@ var ( // DB is the IP geolocation database. type DB struct { file string // Database file name. + checksum string // MD5 of the unzipped database file reader *maxminddb.Reader // Actual db object. notifyQuit chan struct{} // Stop auto-update and watch goroutines. notifyOpen chan string // Notify when a db file is open. @@ -177,7 +178,7 @@ func (db *DB) watchEvents(watcher *fsnotify.Watcher) { } func (db *DB) openFile() error { - reader, err := db.newReader(db.file) + reader, checksum, err := db.newReader(db.file) if err != nil { return err } @@ -185,29 +186,31 @@ func (db *DB) openFile() error { if err != nil { return err } - db.setReader(reader, stat.ModTime()) + db.setReader(reader, stat.ModTime(), checksum) return nil } -func (db *DB) newReader(dbfile string) (*maxminddb.Reader, error) { +func (db *DB) newReader(dbfile string) (*maxminddb.Reader, string, error) { f, err := os.Open(dbfile) if err != nil { - return nil, err + return nil, "", err } defer f.Close() gzf, err := gzip.NewReader(f) if err != nil { - return nil, err + return nil, "", err } defer gzf.Close() b, err := ioutil.ReadAll(gzf) if err != nil { - return nil, err + return nil, "", err } - return maxminddb.FromBytes(b) + checksum := fmt.Sprintf("%x", md5.Sum(b)) + mmdb, err := maxminddb.FromBytes(b) + return mmdb, checksum, err } -func (db *DB) setReader(reader *maxminddb.Reader, modtime time.Time) { +func (db *DB) setReader(reader *maxminddb.Reader, modtime time.Time, checksum string) { db.mu.Lock() defer db.mu.Unlock() if db.closed { @@ -219,6 +222,7 @@ func (db *DB) setReader(reader *maxminddb.Reader, modtime time.Time) { } db.reader = reader db.lastUpdated = modtime.UTC() + db.checksum = checksum select { case db.notifyOpen <- db.file: default: @@ -228,6 +232,7 @@ func (db *DB) setReader(reader *maxminddb.Reader, modtime time.Time) { func (db *DB) autoUpdate(url string) { backoff := time.Second for { + db.sendInfo("starting update") err := db.runUpdate(url) if err != nil { bs := backoff.Seconds() @@ -237,6 +242,7 @@ func (db *DB) autoUpdate(url string) { } else { backoff = db.updateInterval } + db.sendInfo("finished update") select { case <-db.notifyQuit: return @@ -247,7 +253,6 @@ func (db *DB) autoUpdate(url string) { } func (db *DB) runUpdate(url string) error { - db.sendInfo("starting update") yes, err := db.needUpdate(url) if err != nil { return err @@ -264,7 +269,6 @@ func (db *DB) runUpdate(url string) error { // Cleanup the tempfile if renaming failed. os.RemoveAll(tmpfile) } - db.sendInfo("finished update") return err } @@ -273,11 +277,19 @@ func (db *DB) needUpdate(url string) (bool, error) { if err != nil { return true, nil // Local db is missing, must be downloaded. } + resp, err := http.Head(url) if err != nil { return false, err } defer resp.Body.Close() + + // Check X-Database-MD5 if it exists + headerMd5 := resp.Header.Get("X-Database-MD5") + if len(headerMd5) > 0 && db.checksum != headerMd5 { + return true, nil + } + if stat.Size() != resp.ContentLength { return true, nil } @@ -285,7 +297,6 @@ func (db *DB) needUpdate(url string) (bool, error) { } func (db *DB) download(url string) (tmpfile string, err error) { - db.sendInfo("starting download") resp, err := http.Get(url) if err != nil { return "", err @@ -302,7 +313,6 @@ func (db *DB) download(url string) (tmpfile string, err error) { if err != nil { return "", err } - db.sendInfo("finished download") return tmpfile, nil } diff --git a/db_test.go b/db_test.go index d542ecd..e861035 100644 --- a/db_test.go +++ b/db_test.go @@ -92,6 +92,53 @@ func TestNeedUpdateSameFile(t *testing.T) { } } +func TestNeedUpdateSameMD5(t *testing.T) { + db := &DB{file: testFile} + _, checksum, err := db.newReader(db.file) + if err != nil { + t.Fatal(err) + } + db.checksum = checksum + mux := http.NewServeMux() + changeHeaderThenServe := func(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Database-MD5", checksum) + h.ServeHTTP(w, r) + } + } + mux.Handle("/testdata/", changeHeaderThenServe(http.FileServer(http.Dir(".")))) + srv := httptest.NewServer(mux) + defer srv.Close() + yes, err := db.needUpdate(srv.URL + "/" + testFile) + if err != nil { + t.Fatal(err) + } + if yes { + t.Fatal("Unexpected: db is not supposed to need an update") + } +} + +func TestNeedUpdateMD5(t *testing.T) { + mux := http.NewServeMux() + changeHeaderThenServe := func(h http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Database-MD5", "9823y5981y2398y1234") + h.ServeHTTP(w, r) + } + } + mux.Handle("/testdata/", changeHeaderThenServe(http.FileServer(http.Dir(".")))) + srv := httptest.NewServer(mux) + defer srv.Close() + db := &DB{file: testFile} + yes, err := db.needUpdate(srv.URL + "/" + testFile) + if err != nil { + t.Fatal(err) + } + if !yes { + t.Fatal("Unexpected: db is supposed to need an update") + } +} + func TestNeedUpdate(t *testing.T) { mux := http.NewServeMux() mux.Handle("/testdata/", http.FileServer(http.Dir("."))) diff --git a/vendor/vendor.json b/vendor/vendor.json new file mode 100644 index 0000000..a5737fa --- /dev/null +++ b/vendor/vendor.json @@ -0,0 +1,209 @@ +{ + "comment": "", + "heroku": { + "install" : [ "./cmd/..." ], + "goVersion": "go1.7" + }, + "ignore": "test", + "package": [ + { + "checksumSHA1": "spyv5/YFBjYyZLZa1U2LBfDR8PM=", + "path": "github.com/beorn7/perks/quantile", + "revision": "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9", + "revisionTime": "2016-08-04T10:47:26Z" + }, + { + "checksumSHA1": "un79juR7EIavyegPSipMhklDhj4=", + "path": "github.com/bradfitz/gomemcache/memcache", + "revision": "2fafb84a66c4911e11a8f50955b01e74fe3ab9c5", + "revisionTime": "2016-11-27T23:23:02Z" + }, + { + "checksumSHA1": "bvd8LkQAIwPZND7b3vpVsPSbkqQ=", + "path": "github.com/fiorix/freegeoip", + "revision": "2dd1041b59f1341a60bdbecfc4c918a8e86f30b6", + "revisionTime": "2017-01-06T11:01:25Z" + }, + { + "checksumSHA1": "3NOCRqHwvi2qGzdFJpUbuRNvp9s=", + "path": "github.com/fiorix/freegeoip/apiserver", + "revision": "2dd1041b59f1341a60bdbecfc4c918a8e86f30b6", + "revisionTime": "2017-01-06T11:01:25Z" + }, + { + "checksumSHA1": "s3x0QTPmuue1OQckgsD00CvvS4k=", + "path": "github.com/fiorix/freegeoip/cmd/freegeoip", + "revision": "2dd1041b59f1341a60bdbecfc4c918a8e86f30b6", + "revisionTime": "2017-01-06T11:01:25Z" + }, + { + "checksumSHA1": "4aYbMnXo3kz6UgMHC5Vz79BdCaU=", + "path": "github.com/fiorix/go-redis/redis", + "revision": "d987058b55eb470a35e9e18169011cd44bb20cc9", + "revisionTime": "2016-01-04T01:03:33Z" + }, + { + "checksumSHA1": "lwZofErBLqnpq1trzO2hS3Rm1+s=", + "path": "github.com/go-web/httplog", + "revision": "580d0d49f0d3990a37bf9f11f7b14d4021c5d8fc", + "revisionTime": "2016-04-12T23:27:24Z" + }, + { + "checksumSHA1": "qkKztjBfRJc6mMmsX7yg7dR1UR8=", + "path": "github.com/go-web/httpmux", + "revision": "9e95425ee2c3de016d8630f6ec9455aaf7abadd2", + "revisionTime": "2016-05-05T07:02:39Z" + }, + { + "checksumSHA1": "GYxpsIiASzEFuLmpqDJjKB2mWag=", + "path": "github.com/go-web/httprl", + "revision": "20dc8024cb5d04a4bffe6585685122cee1111a9f", + "revisionTime": "2016-05-05T07:01:43Z" + }, + { + "checksumSHA1": "SKzmeIlam7dWKsSBkzIpH11AyVA=", + "path": "github.com/go-web/httprl/memcacherl", + "revision": "20dc8024cb5d04a4bffe6585685122cee1111a9f", + "revisionTime": "2016-05-05T07:01:43Z" + }, + { + "checksumSHA1": "90DPhOc8ocLJ+Sx+QlT1gfM6LaU=", + "path": "github.com/go-web/httprl/redisrl", + "revision": "20dc8024cb5d04a4bffe6585685122cee1111a9f", + "revisionTime": "2016-05-05T07:01:43Z" + }, + { + "checksumSHA1": "kBeNcaKk56FguvPSUCEaH6AxpRc=", + "path": "github.com/golang/protobuf/proto", + "revision": "8ee79997227bf9b34611aee7946ae64735e6fd93", + "revisionTime": "2016-11-17T03:31:26Z" + }, + { + "checksumSHA1": "ZxzYc1JwJ3U6kZbw/KGuPko5lSY=", + "path": "github.com/howeyc/fsnotify", + "revision": "f0c08ee9c60704c1879025f2ae0ff3e000082c13", + "revisionTime": "2015-10-03T19:46:02Z" + }, + { + "checksumSHA1": "Y3U6on66N0BszKXYvb2Q+qpG6f4=", + "path": "github.com/julienschmidt/httprouter", + "revision": "8a45e95fc75cb77048068a62daed98cc22fdac7c", + "revisionTime": "2017-01-04T18:58:16Z" + }, + { + "checksumSHA1": "bKMZjd2wPw13VwoE7mBeSv5djFA=", + "path": "github.com/matttproud/golang_protobuf_extensions/pbutil", + "revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c", + "revisionTime": "2016-04-24T11:30:07Z" + }, + { + "checksumSHA1": "WQJBP9v20jr44RiZ1YbfrpGaEqk=", + "path": "github.com/newrelic/go-agent", + "revision": "7d12ae2201fc160e486197614a6f65afcf3f8170", + "revisionTime": "2016-11-16T22:44:47Z" + }, + { + "checksumSHA1": "lLXXIL0C/ZzMDqN2BlQRZInhot0=", + "path": "github.com/newrelic/go-agent/internal", + "revision": "7d12ae2201fc160e486197614a6f65afcf3f8170", + "revisionTime": "2016-11-16T22:44:47Z" + }, + { + "checksumSHA1": "mkbupMdy+cF7xyo8xW0A6Bq15k4=", + "path": "github.com/newrelic/go-agent/internal/jsonx", + "revision": "7d12ae2201fc160e486197614a6f65afcf3f8170", + "revisionTime": "2016-11-16T22:44:47Z" + }, + { + "checksumSHA1": "ywxlVKtGArJ2vDfH1rAqEFwSGds=", + "path": "github.com/newrelic/go-agent/internal/logger", + "revision": "7d12ae2201fc160e486197614a6f65afcf3f8170", + "revisionTime": "2016-11-16T22:44:47Z" + }, + { + "checksumSHA1": "S7CiHO7EblgZt9q7wgiXMv/j/ao=", + "path": "github.com/newrelic/go-agent/internal/sysinfo", + "revision": "7d12ae2201fc160e486197614a6f65afcf3f8170", + "revisionTime": "2016-11-16T22:44:47Z" + }, + { + "checksumSHA1": "c2JSKesj3tHYgzIF3QL37WfHWG8=", + "path": "github.com/newrelic/go-agent/internal/utilization", + "revision": "7d12ae2201fc160e486197614a6f65afcf3f8170", + "revisionTime": "2016-11-16T22:44:47Z" + }, + { + "checksumSHA1": "rnQM9A55VCOSbv0IOuUn/Yl+IFk=", + "path": "github.com/oschwald/maxminddb-golang", + "revision": "4cf6490e82edd288b91d6a786d85dab042015e24", + "revisionTime": "2016-12-31T00:38:52Z" + }, + { + "checksumSHA1": "/j0HRFJPThv7HEkZZ/gurf+5fQI=", + "path": "github.com/prometheus/client_golang/prometheus", + "revision": "575f371f7862609249a1be4c9145f429fe065e32", + "revisionTime": "2016-11-24T15:57:32Z" + }, + { + "checksumSHA1": "DvwvOlPNAgRntBzt3b3OSRMS2N4=", + "path": "github.com/prometheus/client_model/go", + "revision": "fa8ad6fec33561be4280a8f0514318c79d7f6cb6", + "revisionTime": "2015-02-12T10:17:44Z" + }, + { + "checksumSHA1": "mHyjbJ3BWOfUV6q9f5PBt0gaY1k=", + "path": "github.com/prometheus/common/expfmt", + "revision": "6d76b79f239843a04e8ad8dfd8fcadfa3920236f", + "revisionTime": "2016-12-20T17:45:53Z" + }, + { + "checksumSHA1": "GWlM3d2vPYyNATtTFgftS10/A9w=", + "path": "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg", + "revision": "6d76b79f239843a04e8ad8dfd8fcadfa3920236f", + "revisionTime": "2016-12-20T17:45:53Z" + }, + { + "checksumSHA1": "vopCLXHzYm+3l5fPKOf4/fQwrCM=", + "path": "github.com/prometheus/common/model", + "revision": "6d76b79f239843a04e8ad8dfd8fcadfa3920236f", + "revisionTime": "2016-12-20T17:45:53Z" + }, + { + "checksumSHA1": "L+p4t3KrLDAKJnrreOz2BZIt9Mg=", + "path": "github.com/prometheus/procfs", + "revision": "fcdb11ccb4389efb1b210b7ffb623ab71c5fdd60", + "revisionTime": "2016-12-06T22:21:41Z" + }, + { + "checksumSHA1": "LjPdvMphElL0GOVNQCsmZMVgWIw=", + "path": "github.com/rs/cors", + "revision": "a62a804a8a009876ca59105f7899938a1349f4b3", + "revisionTime": "2016-06-17T23:19:35Z" + }, + { + "checksumSHA1": "hCRfPlNpqv8tvVivLzmXsoUOf1c=", + "path": "github.com/rs/xhandler", + "revision": "ed27b6fd65218132ee50cd95f38474a3d8a2cd12", + "revisionTime": "2016-06-18T19:32:21Z" + }, + { + "checksumSHA1": "9jjO5GjLa0XF/nfWihF02RoH4qc=", + "path": "golang.org/x/net/context", + "revision": "905989bd20b7c354fd28a61074eed1c8f49ebc89", + "revisionTime": "2017-01-06T00:12:52Z" + }, + { + "checksumSHA1": "uTQtOqR0ePMMcvuvAIksiIZxhqU=", + "path": "golang.org/x/sys/unix", + "revision": "d75a52659825e75fff6158388dddc6a5b04f9ba5", + "revisionTime": "2016-12-14T18:38:57Z" + }, + { + "checksumSHA1": "kQB2wRB3twjUp615F6zEwGHjNe0=", + "path": "golang.org/x/sys/windows", + "revision": "d75a52659825e75fff6158388dddc6a5b04f9ba5", + "revisionTime": "2016-12-14T18:38:57Z" + } + ], + "rootPath": "github.com/fiorix/freegeoip" +}