From 4566f4c87d2eabc312bd04c947b6f8007dd7666c Mon Sep 17 00:00:00 2001 From: kristijorgji Date: Wed, 21 Oct 2020 23:45:00 +0200 Subject: [PATCH] initial commit, add main structure and logic to support seeding by environment, seed names, and provide some utils like seed from json file one table, utils only mysql support for now --- .gitignore | 2 + LICENSE | 21 ++++++++++ Makefile | 4 ++ README.md | 44 ++++++++++++++++++++ go.mod | 8 ++++ go.sum | 13 ++++++ helpers.go | 84 +++++++++++++++++++++++++++++++++++++ seeder.go | 109 +++++++++++++++++++++++++++++++++++++++++++++++++ seeder_test.go | 27 ++++++++++++ sources.go | 31 ++++++++++++++ 10 files changed, 343 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 helpers.go create mode 100644 seeder.go create mode 100644 seeder_test.go create mode 100644 sources.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..60ac8bc --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.idea +.env diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d14b26f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Kristi Jorgji + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5468f47 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +test: + go test -v -cover ./... + +.PHONY: test \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..7f3b028 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# goseeder + +#### Motivation +While golang is a great language and getting better, there are still a lot of pieces missing for developing fast and in accurate way to avoid repetitions. + +I was searching for a go seeder similar to the one that Laravel/Lumen provides and could not find one. + +Knowing that this is such an important key element of any big project for testing and seeding projects with dummy data I decided to create one myself and share. + +#### Features +For now the library supports only MySql as a database driver for its utility functions like `FromJson` but is db agnostic for your custom seeders you can use any database that is supported by `sql.DB` + +`goseeder` +1. Allows specifying seeds for different environments such as test,common (all envs) and more (flexible you can define them) +2. Provides out of the box functions like `FromJson` to seed the table from json data and more data formats and drivers coming soon + +# Table of Contents + +- [Installation](#installation) +- [Usage](#usage) +- [License](#license) + +## Installation + +```sh +go get github.com/kristijorgji/goseeder +``` + +## Usage + +To use goseeder you just need to wrap your main function with `WithSeeder` and specify your seeds under a local package named `seeds` in one folder with structure `db/seeds` (it is important if you want to use json or other data to follow this structure) + +// TODO full example usage will be uploaded later in this repo + + +## License + +goseeder is released under the MIT Licence. See the bundled LICENSE file for details. + + + + + + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8f3d87c --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/kristijorgji/goseeder + +go 1.15 + +require ( + github.com/go-sql-driver/mysql v1.5.0 + github.com/stretchr/testify v1.6.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c494b10 --- /dev/null +++ b/go.sum @@ -0,0 +1,13 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..d9bbee4 --- /dev/null +++ b/helpers.go @@ -0,0 +1,84 @@ +package goseeder + +import ( + "fmt" + "reflect" + "runtime" + "strconv" + "strings" +) + +const ( + InfoColor = "\033[1;34m%s\033[0m" + NoticeColor = "\033[1;36m%s\033[0m" + WarningColor = "\033[1;33m%s\033[0m" + ErrorColor = "\033[1;31m%s\033[0m" + DebugColor = "\033[0;36m%s\033[0m" +) + +var printError = color(ErrorColor) + +func color(colorString string) func(...interface{}) string { + return func(args ...interface{}) string { + return fmt.Sprintf(colorString, + fmt.Sprint(args...)) + } +} + +func findString(slice []string, val string) (int, bool) { + for i, item := range slice { + if item == val { + return i, true + } + } + return -1, false +} + +func prepareStatement(table string, row map[string]string) (strings.Builder, []interface{}) { + var left strings.Builder + var right strings.Builder + + var args []interface{} + + left.WriteString(fmt.Sprintf("insert into %s (", table)) + right.WriteString("values (") + + i := 0 + + for k, v := range row { + if i == 0 { + left.WriteString(k) + right.WriteString("?") + } else { + left.WriteString(fmt.Sprintf(", %s", k)) + right.WriteString(", ?") + } + + args = append(args, parseValue(v)) + i++ + } + + left.WriteString(") ") + right.WriteString(")") + left.WriteString(right.String()) + + return left, args +} + +func parseValue(value string) interface{} { + if parsed, err := strconv.ParseInt(value, 10, 64); err == nil { + return parsed + } + if parsed, err := strconv.ParseFloat(value, 32); err == nil { + return parsed + } + if parsed, err := strconv.ParseBool(value); err == nil { + return parsed + } + + return value +} + +func getFunctionName(i interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() +} diff --git a/seeder.go b/seeder.go new file mode 100644 index 0000000..13a8543 --- /dev/null +++ b/seeder.go @@ -0,0 +1,109 @@ +package goseeder + +import ( + "database/sql" + "flag" + "fmt" + "log" + "os" + "regexp" + "strings" + "time" +) + +// Seeder type +type Seeder struct { + DB *sql.DB +} + +type clientSeeder struct { + env string + name string + cb func(s Seeder) +} + +var seeders []clientSeeder + +func WithSeeder(conProvider func() *sql.DB, clientMain func()) { + var seed bool = false + var env string = "" + var names string = "" + + flag.BoolVar(&seed, "gseed", seed, "goseeder - if set will seed") + flag.StringVar(&env, "gsenv", "", "goseeder - env for which seeds to execute") + flag.StringVar(&names, "gsnames", "", "goseeder - comma separated seeder names to run specific ones") + flag.Parse() + + if !seed { + clientMain() + return + } + + var seeders []string = make([]string, 0) + if len(names) > 0 { + seeders = strings.Split(names, ",") + } + + execute(conProvider(), env, seeders...) + os.Exit(0) +} + +func Register(seeder func(s Seeder)) { + RegisterForEnv("", seeder) +} + +func RegisterForTest(seeder func(s Seeder)) { + RegisterForEnv("test", seeder) +} + +func RegisterForEnv(env string, seeder func(s Seeder)) { + r := regexp.MustCompile(`.*\.(?P[a-zA-Z]+$)`) + match := r.FindStringSubmatch(getFunctionName(seeder)) + + seeders = append(seeders, clientSeeder{ + env: env, + name: match[1], + cb: seeder, + }) +} + +// Execute will executes the given seeder method +func execute(db *sql.DB, env string, seedMethodNames ...string) { + s := Seeder{db} + + // Execute all seeders if no method name is given + if len(seedMethodNames) == 0 { + if env == "" { + log.Println("Running all seeders...") + } else { + log.Printf("Running seeders for env %s...\n", env) + } + for _, seeder := range seeders { + if env == "" || env == seeder.env { + seed(&s, seeder) + } + } + return + } + + for _, seeder := range seeders { + if _, r := findString(seedMethodNames, seeder.name); (env == "" || env == seeder.env) && r { + seed(&s, seeder) + } + } +} + +func seed(rootSeeder *Seeder, seeder clientSeeder) { + start := time.Now() + log.Printf("[%s] started seeding...\n", seeder.name) + + defer func() { + if r := recover(); r != nil { + log.Print(printError(fmt.Sprintf("[%s] seed failed: %+v\n", seeder.name, r))) + } + }() + + seeder.cb(*rootSeeder) + elapsed := time.Since(start) + log.Printf("[%s] seeded successfully, duration %s\n", seeder.name, elapsed) +} diff --git a/seeder_test.go b/seeder_test.go new file mode 100644 index 0000000..04aa04b --- /dev/null +++ b/seeder_test.go @@ -0,0 +1,27 @@ +package goseeder + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +var testCases = []struct { + name string + value string + expected interface{} +}{ + {"bool", "true", true}, + {"bool", "false", false}, + {"int", "12", int64(12)}, + {"float", "12.77", 12.770000457763672}, + {"string", "justastring", "justastring"}, + {"string_with_nr", "12justastring", "12justastring"}, +} + +func TestParseValue(t *testing.T) { + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, parseValue(tt.value)) + }) + } +} diff --git a/sources.go b/sources.go new file mode 100644 index 0000000..8ccf219 --- /dev/null +++ b/sources.go @@ -0,0 +1,31 @@ +package goseeder + +import ( + "encoding/json" + "fmt" + _ "github.com/go-sql-driver/mysql" + "io/ioutil" + "log" +) + +func FromJson(s Seeder, filename string) { + content, err := ioutil.ReadFile(fmt.Sprintf("db/seeds/data/%s.json", filename)) + if err != nil { + log.Fatal(err) + } + + m := []map[string]string{} + err = json.Unmarshal(content, &m) + if err != nil { + panic(err) + } + + for _, e := range m { + stmQuery, args := prepareStatement(filename, e) + stmt, _ := s.DB.Prepare(stmQuery.String()) + _, err := stmt.Exec(args...) + if err != nil { + panic(err) + } + } +}