0.8.2 memory leak fixed + geolocation crash fixed + docs errata

README.md

@@ -1,11 +1,11 @@
 
-![Version](https://img.shields.io/badge/version-0.8.1-orange.svg)  
+![Version](https://img.shields.io/badge/version-0.8.2-orange.svg)  
 ![Maintained YES](https://img.shields.io/badge/Maintained%3F-yes-green.svg)  
 ![Ask Me Anything !](https://img.shields.io/badge/Ask%20me-anything-1abc9c.svg)  
 
 # ![logo](https://github.com/jolav/codetabs/blob/master/www/_public/icons/ct/ct64r.png?raw=true) **ONLINE TOOLS ([codetabs.com](https://codetabs.com))** 
 
-**version 0.8.1**
+**version 0.8.2**
 
 1. [Count LOC (lines of code) online from github/gitlab repos or zipped uploaded folder](#count-loc-online)  
 2. [CORS proxy](#cors-proxy)  
@@ -238,9 +238,9 @@ Scale : Set width:height , if one parameter is -1 it will automatically determin
 - [ ] **WWW** clean unused parts, css, etc
 - [ ] **WWW** change web design
 
-- [ ] **ALL** Update Tests  
+- [ ] **ALL** Fix the tests. They are outdated and unusable.
 
-- [ ] **LOC** Save Historical Data  
+- [X] **LOC** Save Historical Data  
 - [X] **LOC** Gitlab  
 - [ ] **LOC** Bitbucket  
 - [X] **LOC** Use same colours for languages as github   
@@ -250,7 +250,7 @@ Scale : Set width:height , if one parameter is -1 it will automatically determin
 - [X] **LOC** box to ignore patterns such as ./vendor  
 - [ ] **LOC** update line count when hiding languages  
 
-- [ ] **STARS** Save Historical Data (unstar = problem)  
+- [X] **STARS** Save Historical Data (unstar = problem)  
 - [X] **STARS** Gitlab  
 - [ ] **STARS** Optimize doing far fewer requests. Extrapolate data  
 

alexa/alexa.go

@@ -23,10 +23,11 @@ const (
 	DATA_FILE_URL       = "https://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
 )
 
+var a alexa
+
 type alexa struct {
 	config    config
 	alexaList map[string]int
-	output    output
 }
 
 type config struct {
@@ -36,12 +37,12 @@ type config struct {
 	DataFileURL  string
 }
 
-type output struct {
+type outputAlexa struct {
 	Web  string `json:"web"`
 	Rank int    `json:"rank"`
 }
 
-func (a *alexa) Router(w http.ResponseWriter, r *http.Request) {
+func Router(w http.ResponseWriter, r *http.Request) {
 	params := strings.Split(strings.ToLower(r.URL.Path), "/")
 	path := params[1:len(params)]
 	if path[len(path)-1] == "" { // remove last empty slot after /
@@ -52,38 +53,41 @@ func (a *alexa) Router(w http.ResponseWriter, r *http.Request) {
 		u.BadRequest(w, r)
 		return
 	}
+	oa := outputAlexa{
+		Web:  "",
+		Rank: 0,
+	}
 	r.ParseForm()
-	web := r.Form.Get("web")
-	if web == "" {
+	oa.Web = r.Form.Get("web")
+	if oa.Web == "" {
 		u.BadRequest(w, r)
 		return
 	}
-	a.doAlexaRequest(w, web)
+	oa.doAlexaRequest(w)
 }
 
-func (a *alexa) doAlexaRequest(w http.ResponseWriter, web string) {
-	a.output.Web = web
-	a.output.Rank = a.alexaList[a.output.Web]
-	if a.output.Rank != 0 {
-		u.SendJSONToClient(w, a.output, 200)
+func (oa *outputAlexa) doAlexaRequest(w http.ResponseWriter) {
+	oa.Rank = a.alexaList[oa.Web]
+	if oa.Rank != 0 {
+		u.SendJSONToClient(w, oa, 200)
 		return
 	}
-	if strings.HasPrefix(a.output.Web, "www.") {
-		a.output.Web = a.output.Web[4:len(a.output.Web)]
-		a.output.Rank = a.alexaList[a.output.Web]
-		if a.output.Rank != 0 {
-			u.SendJSONToClient(w, a.output, 200)
+	if strings.HasPrefix(oa.Web, "www.") {
+		oa.Web = oa.Web[4:len(oa.Web)]
+		oa.Rank = a.alexaList[oa.Web]
+		if oa.Rank != 0 {
+			u.SendJSONToClient(w, oa, 200)
 			return
 		}
 	}
-	if !strings.HasPrefix(a.output.Web, "www.") {
-		a.output.Rank = a.alexaList["www."+a.output.Web]
-		if a.output.Rank != 0 {
-			u.SendJSONToClient(w, a.output, 200)
+	if !strings.HasPrefix(oa.Web, "www.") {
+		oa.Rank = a.alexaList["www."+oa.Web]
+		if oa.Rank != 0 {
+			u.SendJSONToClient(w, oa, 200)
 			return
 		}
 	}
-	msg := fmt.Sprintf("%s not in alexa top 1 million", a.output.Web)
+	msg := fmt.Sprintf("%s not in alexa top 1 million", oa.Web)
 	u.ErrorResponse(w, msg)
 	return
 }
@@ -108,7 +112,8 @@ func (a *alexa) loadDataInMemory() {
 	}
 }
 
-func (a *alexa) OnceADayTask() {
+func OnceADayTask() {
+	a = newAlexa(false)
 	t := time.Now()
 	n := time.Date(t.Year(), t.Month(), t.Day(), 3, 10, 10, 0, t.Location())
 	d := n.Sub(t)
@@ -157,7 +162,7 @@ func (a *alexa) unzipCsv() {
 	}
 }
 
-func NewAlexa(test bool) alexa {
+func newAlexa(test bool) alexa {
 	a := alexa{
 		config: config{
 			DataFilePath: DATA_FILE_PATH,
@@ -166,10 +171,6 @@ func NewAlexa(test bool) alexa {
 			DataFileURL:  DATA_FILE_URL,
 		},
 		alexaList: make(map[string]int),
-		output: output{
-			Web:  "",
-			Rank: 0,
-		},
 	}
 	if test {
 		a.config.DataFilePath = DATA_FILE_PATH_TEST

alexa/alexa_test.go

@@ -16,7 +16,7 @@ func TestAlexaApi(t *testing.T) {
 		validFormat = "Bad request, valid format is 'api.codetabs.com/v1/{service}?{param}=value' .Please read our docs at https://codetabs.com"
 	)
 
-	a := NewAlexa(true)
+	//a := NewAlexa(true)
 
 	type alexaTestOutput struct {
 		Domain     string `json:"domain"`
@@ -65,7 +65,7 @@ func TestAlexaApi(t *testing.T) {
 		}
 		if pass {
 			rr := httptest.NewRecorder()
-			handler := http.HandlerFunc(a.Router)
+			handler := http.HandlerFunc(Router)
 
 			handler.ServeHTTP(rr, req)
 			if rr.Code != test.statusCode {

geolocation/geolocation.go

@@ -39,14 +39,14 @@ type geoData struct {
 	Longitude    float32 `json:"longitude"`
 }
 
-func (g *geoip) Router(w http.ResponseWriter, r *http.Request) {
+func Router(w http.ResponseWriter, r *http.Request) {
 	params := strings.Split(strings.ToLower(r.URL.Path), "/")
 	path := params[1:len(params)]
 	if path[len(path)-1] == "" { // remove last empty slot after /
 		path = path[:len(path)-1]
 	}
 	//log.Printf("Going ....%s %s %d", path, r.Method, len(path))
-	if len(path) < 2 || path[0] != "v1" {
+	if len(path) < 3 || path[0] != "v1" {
 		u.BadRequest(w, r)
 		return
 	}
@@ -55,6 +55,7 @@ func (g *geoip) Router(w http.ResponseWriter, r *http.Request) {
 		u.BadRequest(w, r)
 		return
 	}
+	g := newGeoLocation(false)
 	r.ParseForm()
 	target := strings.ToLower(r.Form.Get("q"))
 	if target == "" {
@@ -64,7 +65,7 @@ func (g *geoip) Router(w http.ResponseWriter, r *http.Request) {
 }
 
 func (g *geoip) doGeoRequest(w http.ResponseWriter, format, target string) {
-	g.cleanGeoData()
+	//g.cleanGeoData()
 	addr, err := net.LookupIP(target)
 	if err != nil {
 		msg := fmt.Sprintf("%s is a unknown host, not a valid IP or hostname", target)
@@ -84,13 +85,13 @@ func (g *geoip) getGeoDataFromDB() {
 	db, err := ip2location.OpenDB(g.config.dbFilePath)
 	if err != nil {
 		log.Println("ERROR GEOIP 1 =", err)
-		g.cleanGeoData()
+		//g.cleanGeoData()
 		return
 	}
 	results, err := db.Get_all(g.geoData.Ip)
 	if err != nil {
 		log.Println("ERROR GEOIP 2 =", err)
-		g.cleanGeoData()
+		//g.cleanGeoData()
 		return
 	}
 	//u.PrettyPrintStruct(results)
@@ -109,7 +110,7 @@ func (g *geoip) cleanGeoData() {
 	g.geoData = geoData{}
 }
 
-func NewGeoLocation(test bool) geoip {
+func newGeoLocation(test bool) geoip {
 	g := geoip{
 		config: config{
 			dbFilePath: DB_FILE_PATH,

geolocation/geolocation_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestGeoipApi(t *testing.T) {
-	g := NewGeoLocation(true)
+	//g := newGeoLocation(true)
 
 	for _, test := range geoipTests {
 		var to = geoipTestOutput{}
@@ -32,7 +32,7 @@ func TestGeoipApi(t *testing.T) {
 		}
 		if pass {
 			rr := httptest.NewRecorder()
-			handler := http.HandlerFunc(g.Router)
+			handler := http.HandlerFunc(Router)
 			handler.ServeHTTP(rr, req)
 			if rr.Code != test.statusCode {
 				t.Errorf("%s got %v want %v\n", test.endpoint, rr.Code, test.statusCode)

headers/headers.go

@@ -10,11 +10,16 @@ import (
 	u "github.com/jolav/codetabs/_utils"
 )
 
+type headersRequest struct {
+	headers
+	domain string
+}
+
 type headers []header
 
 type header map[string]string
 
-func (h *headers) Router(w http.ResponseWriter, r *http.Request) {
+func Router(w http.ResponseWriter, r *http.Request) {
 	params := strings.Split(strings.ToLower(r.URL.Path), "/")
 	path := params[1:len(params)]
 	if path[len(path)-1] == "" { // remove last empty slot after /
@@ -25,17 +30,18 @@ func (h *headers) Router(w http.ResponseWriter, r *http.Request) {
 		u.BadRequest(w, r)
 		return
 	}
+	hr := newHeadersRequest(false)
 	r.ParseForm()
-	domain := r.Form.Get("domain")
-	if domain == "" || len(path) != 2 {
+	hr.domain = r.Form.Get("domain")
+	if hr.domain == "" || len(path) != 2 {
 		u.BadRequest(w, r)
 		return
 	}
-	doHeadersRequest(w, r, domain)
+	hr.doHeadersRequest(w, r)
 }
 
-func doHeadersRequest(w http.ResponseWriter, r *http.Request, domain string) {
-	hs := headers{}
+func (hr *headersRequest) doHeadersRequest(
+	w http.ResponseWriter, r *http.Request) {
 	notMoreRedirections := false
 	count := 0
 	const curl = "curl -fsSI "
@@ -46,28 +52,28 @@ func doHeadersRequest(w http.ResponseWriter, r *http.Request, domain string) {
 	}
 
 	for !notMoreRedirections && count < 10 {
-		rawData, err := u.GenericCommandSH(curl + domain)
+		rawData, err := u.GenericCommandSH(curl + hr.domain)
 		if err != nil {
 			msg := fmt.Sprintf("ERROR %s -> %s %s",
 				r.URL.RequestURI(),
 				curlStatus[err.Error()],
-				domain,
+				hr.domain,
 			)
 			u.ErrorResponse(w, msg)
 			return
 		}
-		parseHeadString(string(rawData), &hs)
-		if hs[count]["Location"] == "" {
+		parseHeadString(string(rawData), &hr.headers)
+		if hr.headers[count]["Location"] == "" {
 			notMoreRedirections = true
 			//fmt.Println(`No more redirections`)
 		} else {
-			domain = hs[count]["Location"]
+			hr.domain = hr.headers[count]["Location"]
 			//fmt.Println(`Redirecting to ... `, domain)
 		}
 		count++
 	}
 
-	u.SendJSONToClient(w, hs, 200)
+	u.SendJSONToClient(w, hr.headers, 200)
 	return
 }
 
@@ -100,7 +106,10 @@ func parseHeadString(rawData string, hs *headers) {
 	*hs = append(*hs, myheader)
 }
 
-func NewHeaders(test bool) headers {
-	h := headers{}
-	return h
+func newHeadersRequest(test bool) headersRequest {
+	hr := headersRequest{
+		headers: []header{},
+		domain:  "",
+	}
+	return hr
 }

loc/loc.go

@@ -59,7 +59,7 @@ type sourceReader interface {
 	exceedsSize(http.ResponseWriter, *loc) bool
 }
 
-func (l *loc) Router(w http.ResponseWriter, r *http.Request) {
+func Router(w http.ResponseWriter, r *http.Request) {
 	params := strings.Split(strings.ToLower(r.URL.Path), "/")
 	path := params[1:len(params)]
 	if path[len(path)-1] == "" { // remove last empty slot after /
@@ -70,15 +70,8 @@ func (l *loc) Router(w http.ResponseWriter, r *http.Request) {
 		u.BadRequest(w, r)
 		return
 	}
-	// clean
-	l.repo = ""
-	l.branch = ""
-	l.ignored = []string{}
-	l.source = ""
-	l.date = ""
-	l.size = 0
-	l.languagesIN = []languageIN{}
-	l.languagesOUT = []languageOUT{}
+
+	l := newLoc(false)
 
 	if r.Method == "POST" {
 		l.orderInt++
@@ -349,7 +342,7 @@ func (l *loc) storeData() {
 	//go store.SaveDataLoc(d)
 }
 
-func NewLoc(test bool) loc {
+func newLoc(test bool) loc {
 	l := loc{
 		order:        "0",
 		orderInt:     0,