0.8.0 database to save loc + stars

.gitignore

@@ -14,6 +14,8 @@ _private.js
 _private.json
 a_private.go
 banned.list
+codetabsData.db3
+*.db3
 # 
 
 # 

README.md

@@ -1,11 +1,11 @@
 
-![Version](https://img.shields.io/badge/version-0.7.20-orange.svg)  
+![Version](https://img.shields.io/badge/version-0.8.0-orange.svg)  
 ![Maintained YES](https://img.shields.io/badge/Maintained%3F-yes-green.svg)  
 ![Ask Me Anything !](https://img.shields.io/badge/Ask%20me-anything-1abc9c.svg)  
 
 # ![logo](https://github.com/jolav/codetabs/blob/master/www/_public/icons/ct/ct64r.png?raw=true) **ONLINE TOOLS ([codetabs.com](https://codetabs.com))** 
 
-**version 0.7.20**
+**version 0.8.0**
 
 1. [Count LOC (lines of code) online from github/gitlab repos or zipped uploaded folder](#count-loc-online)  
 2. [CORS proxy](#cors-proxy)  

geolocation/geolocation.go

@@ -93,7 +93,7 @@ func (g *geoip) getGeoDataFromDB() {
 		g.cleanGeoData()
 		return
 	}
-	u.PrettyPrintStruct(results)
+	//u.PrettyPrintStruct(results)
 	g.geoData.Country_code = results.Country_short
 	g.geoData.Country_name = results.Country_long
 	g.geoData.Region_name = results.Region

go.mod

@@ -2,4 +2,7 @@ module github.com/jolav/codetabs
 
 go 1.14
 
-require github.com/ip2location/ip2location-go/v9 v9.1.0
+require (
+	github.com/ip2location/ip2location-go/v9 v9.1.0
+	github.com/mattn/go-sqlite3 v1.14.16 // indirect
+)

go.sum

@@ -1,2 +1,4 @@
 github.com/ip2location/ip2location-go/v9 v9.1.0 h1:DVlKxIcjA7CsA0cgzbidwe6eKzpbkS313LsH9ceutxI=
 github.com/ip2location/ip2location-go/v9 v9.1.0/go.mod h1:s5SV6YZL10TpfPpXw//7fEJC65G/yH7Oh+Tjq9JcQEQ=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=

loc/loc.go

@@ -11,8 +11,10 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	u "github.com/jolav/codetabs/_utils"
+	"github.com/jolav/codetabs/store"
 )
 
 const (
@@ -84,6 +86,7 @@ func (l *loc) Router(w http.ResponseWriter, r *http.Request) {
 		l.doLocUploadRequest(w, r)
 		return
 	}
+
 	r.ParseForm()
 	l.branch = r.Form.Get("branch")
 	if r.Form.Get("ignored") != "" {
@@ -134,6 +137,7 @@ func (l *loc) doLocRepoRequest(w http.ResponseWriter, r *http.Request) {
 			total2.Lines += v.Lines
 		}
 		l.languagesOUT = append(l.languagesOUT, total2)
+		l.storeData()
 		u.SendJSONToClient(w, l.languagesOUT, 200)
 		return
 	*/
@@ -218,7 +222,12 @@ func (l *loc) doLocRepoRequest(w http.ResponseWriter, r *http.Request) {
 		total.Lines += v.Lines
 	}
 	l.languagesOUT = append(l.languagesOUT, total)
-
+	if l.branch == "" { // cant call gitlab with master
+		l.branch = "master"
+	}
+	if len(l.ignored) == 0 && l.branch == "master" {
+		l.storeData()
+	}
 	u.SendJSONToClient(w, l.languagesOUT, 200)
 	u.GenericCommand(destroyTemporalDir)
 }
@@ -317,11 +326,29 @@ func (l *loc) doLocUploadRequest(w http.ResponseWriter, r *http.Request) {
 		total.Lines += v.Lines
 	}
 	l.languagesOUT = append(l.languagesOUT, total)
-
+	//l.storeData()
 	u.SendJSONToClient(w, l.languagesOUT, 200)
 	u.GenericCommand(destroyTemporalDir)
 }
 
+func (l *loc) storeData() {
+	d := store.NewDataLoc()
+
+	dataJSON, err := json.Marshal(l.languagesOUT)
+	if err != nil {
+		log.Printf("ERROR Marshaling %s\n", err)
+		d.Data = string(`{}`)
+	} else {
+		d.Data = string(dataJSON)
+	}
+	d.Date = time.Now().Format("2006-01-02 15:04:05.000")
+	d.Repo = l.source + "/" + l.repo
+	d.Source = l.source
+	//u.PrettyPrintStruct(d)
+	go d.SaveDataLoc()
+	//go store.SaveDataLoc(d)
+}
+
 func NewLoc(test bool) loc {
 	l := loc{
 		order:        "0",

main.go

@@ -20,11 +20,12 @@ import (
 	lo "github.com/jolav/codetabs/loc"
 	px "github.com/jolav/codetabs/proxy"
 	st "github.com/jolav/codetabs/stars"
+	"github.com/jolav/codetabs/store"
 	vg "github.com/jolav/codetabs/video2gif"
 	we "github.com/jolav/codetabs/weather"
 )
 
-var version = "0.7.20"
+var version = "0.8.0"
 var when = "undefined"
 
 type Conf struct {
@@ -45,6 +46,13 @@ func main() {
 	checkMode(&c)
 	//u.PrettyPrintStruct(c)
 
+	// Database
+	db, err := store.NewDB()
+	if err != nil {
+		log.Fatal("Error connecting DataBase => ", err)
+	}
+	store.MyDB = db
+
 	// Custom Error Log File + Custom Hits Log File
 	var mylog *os.File
 	if c.Mode == "production" {
@@ -101,10 +109,31 @@ func mw(next http.HandlerFunc, service string, c Conf) http.HandlerFunc {
 				return
 			}
 		}
+
 		next.ServeHTTP(w, r)
 	})
 }
 
+/*
+mux.HandleFunc("/vnstats/v1/", checkValid(
+	func(w http.ResponseWriter, r *http.Request) {
+		vnstatsRouter(w, r, a)
+	}, a.c.APIKey),
+)
+
+func checkValid(next http.HandlerFunc, test string) http.HandlerFunc {
+return func(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	valid := r.Form.Get("test")
+	if valid != test {
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		return
+	}
+	next.ServeHTTP(w, r)
+}
+}
+*/
+
 func checkFlags() {
 	versionFlag := flag.Bool("v", false, "Show current version and exit")
 	flag.Parse()

stars/github.go

@@ -27,6 +27,7 @@ func (s *stars) doGHStarsRequest(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	if totalStars == 0 { // repo exists but has no stars
+		s.storeData()
 		u.SendJSONToClient(w, s.data, 200)
 		return
 	}
@@ -47,6 +48,7 @@ func (s *stars) doGHStarsRequest(w http.ResponseWriter, r *http.Request) {
 		last.Quantity = totalStars
 		s.data = append(s.data, last)
 	}
+	s.storeData()
 	u.SendJSONToClient(w, s.data, 200)
 }
 

stars/gitlab.go

@@ -27,6 +27,7 @@ func (s *stars) doGLStarsRequest(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	if totalStars == 0 { // repo exists but has no stars
+		s.storeData()
 		u.SendJSONToClient(w, s.data, 200)
 		return
 	}
@@ -36,6 +37,7 @@ func (s *stars) doGLStarsRequest(w http.ResponseWriter, r *http.Request) {
 	if len(s.stars) > 0 {
 		s.convertGLStarsToData()
 	}
+	s.storeData()
 	u.SendJSONToClient(w, s.data, 200)
 }
 

stars/stars.go

@@ -3,12 +3,15 @@
 package stars
 
 import (
+	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"strings"
 	"time"
 
 	u "github.com/jolav/codetabs/_utils"
+	"github.com/jolav/codetabs/store"
 )
 
 type star struct {
@@ -79,6 +82,24 @@ func (s *stars) Router(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *stars) storeData() {
+	d := store.NewDataStars()
+
+	dataJSON, err := json.Marshal(s.data)
+	if err != nil {
+		log.Printf("ERROR Marshaling %s\n", err)
+		d.Data = string(`{}`)
+	} else {
+		d.Data = string(dataJSON)
+	}
+	d.Date = time.Now().Format("2006-01-02 15:04:05.000")
+	d.Repo = s.source + "/" + s.repo
+	d.Source = s.source
+	d.Total = len(s.stars)
+	//u.PrettyPrintStruct(d)
+	go d.SaveDataStars()
+}
+
 func (s *stars) cleanStarsStruct() {
 	s.stars = []*star{}
 	s.headerLink = ""

store/loc.go

@@ -0,0 +1,68 @@
+/* */
+
+package store
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"time"
+
+	u "github.com/jolav/codetabs/_utils"
+)
+
+type dataLoc struct {
+	Repo   string `json:"repo"`
+	Source string `json:"source"`
+	Date   string `json:"date"`
+	Data   string `json:"data"`
+}
+
+func (d dataLoc) SaveDataLoc() {
+	d.replaceLoc()
+	//LoadDataLoc(d)
+}
+
+func (d dataLoc) replaceLoc() {
+	sql := fmt.Sprintf("REPLACE INTO %s (repo, source, date, data) VALUES (?, ?, ?, ?)",
+		locTable)
+	stmt, err := MyDB.Prepare(sql)
+	if err != nil {
+		log.Printf("Error DB 1 replaceLoc => %s\n", err)
+		return
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec(d.Repo, d.Source, d.Date, d.Data)
+	if err != nil {
+		log.Printf("Error DB 2 replaceLoc => %s\n", err)
+		return
+	}
+}
+
+func (d dataLoc) LoadDataLoc() {
+	type languageOUT struct {
+		Name     string `json:"language"`
+		Files    int    `json:"files"`
+		Lines    int    `json:"lines"`
+		Blanks   int    `json:"blanks"`
+		Comments int    `json:"comments"`
+		Code     int    `json:"linesOfCode"`
+	}
+	var languagesOUT []languageOUT
+	err := json.Unmarshal([]byte(d.Data), &languagesOUT)
+	if err != nil {
+		log.Printf("ERROR LoadConfig %s\n", err)
+	}
+	u.PrettyPrintStruct(languagesOUT)
+}
+
+func NewDataLoc() dataLoc {
+	d := dataLoc{
+		Repo:   "",
+		Source: "",
+		Date:   time.Now().Format("2006-01-02 15:04:05.000"),
+		Data:   "{}",
+	}
+	return d
+}

store/stars.go

@@ -0,0 +1,67 @@
+/* */
+
+package store
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"time"
+
+	u "github.com/jolav/codetabs/_utils"
+)
+
+type dataStars struct {
+	Repo   string `json:"repo"`
+	Source string `json:"source"`
+	Date   string `json:"date"`
+	Total  int    `json:"total"`
+	Data   string `json:"data"`
+}
+
+func (d dataStars) SaveDataStars() {
+	d.replaceStars()
+	//LoadDataStars(d)
+
+}
+
+func (d dataStars) replaceStars() {
+	sql := fmt.Sprintf("REPLACE INTO %s (repo, source, date, total, data) VALUES (?, ?, ?, ?, ?)",
+		starsTable)
+	stmt, err := MyDB.Prepare(sql)
+	if err != nil {
+		log.Printf("Error DB 1 replaceStars => %s\n", err)
+		return
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec(d.Repo, d.Source, d.Date, d.Total, d.Data)
+	if err != nil {
+		log.Printf("Error DB 2 replaceStars => %s\n", err)
+		return
+	}
+}
+
+func (d dataStars) LoadDataStars() {
+	type point struct {
+		When     string `json:"x"`
+		Quantity int    `json:"y"`
+	}
+	var data []point
+	err := json.Unmarshal([]byte(d.Data), &data)
+	if err != nil {
+		log.Printf("ERROR LoadConfig %s\n", err)
+	}
+	u.PrettyPrintStruct(data)
+}
+
+func NewDataStars() dataStars {
+	d := dataStars{
+		Repo:   "",
+		Source: "",
+		Date:   time.Now().Format("2006-01-02 15:04:05.000"),
+		Total:  0,
+		Data:   "{}",
+	}
+	return d
+}