From 7cff05835942fddf725b743db670fa9429732165 Mon Sep 17 00:00:00 2001 From: kristijorgji Date: Tue, 19 Mar 2024 18:22:33 +0100 Subject: [PATCH] add RegisterForEnvNamed named method which registers seed and allows setting a specific name. fromJson stringify maps and allow for reading json content from a particular column. Readme update. Print also seed environment during progress or error --- README.md | 98 ++++++++++++------- .../db/migrations/000001_init_schema.up.sql | 8 +- helpers.go | 7 ++ seeder.go | 17 +++- seeder_test.go | 8 ++ sources.go | 11 ++- 6 files changed, 105 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 3f64cfd..595ef7a 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,16 @@ I was searching for a go seeder similar to the one that Laravel/Lumen provides a Knowing that this is such an important key element of any big project for testing and seeding projects with dummy data I decided to create one myself and share. #### Features -For now the library supports only MySql as a database driver for its utility functions like `FromJson` provided by `Seeder` struct, but it is db agnostic for your custom seeders you can use any database that is supported by `sql.DB` +For now the library supports only MySql as a database driver for its utility functions like `FromJson` +and `FromJsonIntoTable` provided by `Seeder` struct, but it is db agnostic for your custom seeders you can use any +database that is supported by `sql.DB` `goseeder` 1. It is designed for different kind of usages, for both programmatically or building into your exe and run via cli args -1. Allows specifying seeds for different environments such as predefined test and custom specified envs by the user -2. Allows specifying list (or single) seed name for execution -3. Allows having common seeds that execute for every env, unless specified not to do so with the respective cli or option -4. Provides out of the box functions like `(s Seeder) FromJson` to seed the table from json data and more data formats and drivers coming soon +2. Allows specifying seeds for different environments such as predefined test and custom specified envs by the user +3. Allows specifying list (or single) seed name for execution +4. Allows having common seeds that execute for every env, unless specified not to do so with the respective cli or option +5. Provides out of the box functions like `(s Seeder) FromJson` to seed the table from json data and more data formats and drivers coming soon # Table of Contents @@ -31,7 +33,7 @@ For now the library supports only MySql as a database driver for its utility fun - [2. Registering Your Seeds](#2-registering-your-seeds) - [3. Run Seeds Only For Specific Env](#3-run-seeds-only-for-specific-env) - [4. Run Seeds By Name](#4-run-seeds-by-name) -- [Usage method 2: Programmatically](#usage-method-2-programmatically) +- [Usage method 2: Programmatically](#usage-method-2-programmatically) - [Summary Of Cli Args](#summary-of-cli-args) - [License](#license) @@ -126,13 +128,15 @@ Great! After step 1 our executable is able to run in seed mode or default mode. Now we want to know how to register our custom seeds. If you look at the imports in main file from step one, we might notice that we import -`_ "simpleshop/db/seeds"` even though we do not use them directly. +`_ "simpleshop/db/seeds"` even though we do not use them directly. This is mandatory because our seeds will get registered during package initialisation as we will see later. -The recommended project folder structure to work properly with `goseeder` is to have the following path for the seeders `db/seeds` and the package name to be `seeds` +The recommended project folder structure to work properly with `goseeder` is to have the following path for the +seeders `db/seeds` and the package name to be `seeds` -Inside the folder you can add your seeders, for example lets seed some data into the `categories` table from one json file located at `db/seeds/data/categories.json`. +Inside the folder you can add your seeders, for example lets seed some data into the `categories` table from one json +file located at `db/seeds/data/categories.json`. To do that we create our `categories.go` file at `db/seeds` folder: @@ -158,7 +162,9 @@ Seeds can be registered as: We are going to create below a seed that runs for all environments, so we will not specify any env while registering it. -To do that we create in the `db/seeds` folder the file `common.go` that will register seeds that get always executed regardless of the environment: +To do that we create in the `db/seeds` folder the file `common.go` that will register seeds that get always executed +regardless of the environment: + ```go // db/seeds/common.go package seeds @@ -172,20 +178,23 @@ func init() { We used `goseeder.Register` to register our seed function to run for all environments. -That is all for the basic usage of goseeder!!! +That is all for the basic usage of goseeder!!! Our function in categories.go file is now registered and ready to be used. Now you can run + ``` go run main.go --gseed ``` and it will run all your seeds against the provided db connection. -The framework will look for `categories.json` file in the path `db/seeds/data`, and insert all the entries there in a table named `categories` (inferred from the file name) +The framework will look for `categories.json` file in the path `db/seeds/data`, and insert all the entries there in a +table named `categories` (inferred from the file name) -If you have a seed registered for another environment, for example a test seed, the framework instead will look for the json file at `db/seeds/data/test` +If you have a seed registered for another environment, for example a test seed, the framework instead will look for the +json file at `db/seeds/data/test` So the rule is it will always lookup in this pattern `db/seeds/data/[environment]/[specifiedFileName].[type]` @@ -208,14 +217,20 @@ func init() { ### 3. Run Seeds Only For Specific Env -Many times we want to have seeds only for `test` environment, test purpose and want to avoid having thousand of randomly generated rows inserted into production database by mistake! +Many times we want to have seeds only for `test` environment, test purpose and want to avoid having thousand of randomly +generated rows inserted into production database by mistake! -Or we just want to have granular control, to have separate data to populate our app/web in different way for `staging` `prod` `yourcustomenv` and so on. +Or we just want to have granular control, to have separate data to populate our app/web in different way +for `staging` `prod` `yourcustomenv` and so on. goseeder is designed to take care of this by using one of the following methods: - `goseeder.RegisterForTest(seeder func(s Seeder)` - registers the specified seed for the env named `test` -- `goseeder.RegisterForEnv(env string, seeder func(s Seeder))` - will register your seeder to be executed only for the custom specified env +- `goseeder.RegisterForEnv(env string, seeder func(s Seeder))` - will register your seeder to be executed only for the + custom specified env +- `goseeder.RegisterForEnvNamed(env string, seeder func(s Seeder), name string)` - will register your seeder to be + executed only for the custom specified env with the given name. That name is used during progress output and in case + of errors Let's add to our previous categories.go seeder one seed function specific only for test env! The file now will look like: @@ -245,7 +260,8 @@ func testCategoriesSeeder(s goseeder.Seeder) { ``` -Finally, lets create our registrator file for all test seeds same way as we did with `common.go`, we will create `test.go` now with content as below: +Finally, lets create our registrator file for all test seeds same way as we did with `common.go`, we will +create `test.go` now with content as below: ```go // db/seeds/test.go @@ -259,9 +275,10 @@ func init() { ``` -That is all! +That is all! -Now if you run your app without specifying test env, only the common env seeders will run and you cannot mess by mistake production or other environments! +Now if you run your app without specifying test env, only the common env seeders will run and you cannot mess by mistake +production or other environments! To run the test seeder above you have to run: @@ -269,7 +286,8 @@ To run the test seeder above you have to run: go run main.go --gseed --gsenv=test ``` -This will run only the tests registered for the env `test` and the `common` seeds. A seed is known as common if it is registered without envrionment via `Register` method and has empty string env. +This will run only the tests registered for the env `test` and the `common` seeds. A seed is known as common if it is +registered without environment via `Register` method and has empty string env. If you do not want common seeds to get executed, just specify the flag `--gs-skip-common` @@ -281,7 +299,8 @@ go run main.go --gseed --gsenv=test --gs-skip-common ### 4. Run Seeds By Name -When we register a seed like shown in step 2, the seed name is the same as the function name, so our seed is called `categoriesSeeder` because that is the name of the function we register below +When we register a seed like shown in step 2, the seed name is the same as the function name, so our seed is +called `categoriesSeeder` because that is the name of the function we register below ```go func init() { @@ -289,9 +308,11 @@ func init() { } ``` -This is important because we are totally flexible and can do cool things like execute only the specific seed functions that we want! +This is important because we are totally flexible and can do cool things like execute only the specific seed functions +that we want! -Let's assume that we have 100 seed functions, and want to execute only one of them which is named categoriesSeeder (that we registered above) and ignore all the other seeds. +Let's assume that we have 100 seed functions, and want to execute only one of them which is named categoriesSeeder (that +we registered above) and ignore all the other seeds. Easy as this, just run: @@ -299,13 +320,14 @@ Easy as this, just run: go run main.go --gseeder --gsnames=categoriesSeeder ``` -If you want to execute multiple seeds by specifying their names, just use comma separated value like `--gsnames=categoriesSeeder,someOtherSeeder` +If you want to execute multiple seeds by specifying their names, just use comma separated value +like `--gsnames=categoriesSeeder,someOtherSeeder` ## Usage method 2: Programmatically `goseeder` is designed to fit all needs for being the best seeding tool. -That means that you might want to seed data before your unit tests programmatically without using cli args. +That means that you might want to seed data before your unit tests programmatically without using cli args. That is straightforward to do with goseeder. Let us assume we want to test our api that connects to some database in the package `api`, @@ -349,12 +371,15 @@ How nice is that ?! After the execution you have a database with data sourcing only from your seeds registered for test env (test seeds) !!! The above is production code used by one company, but you might need to adjust to your needs. -Another common use case is to want to execute programmatically the seeder because you don't want to turn your executable into seedable (you don't want to use method 1)). +Another common use case is to want to execute programmatically the seeder because you don't want to turn your executable +into seedable (you don't want to use method 1)). -Then again you can just create another file `myseeder.go` and inside it do your custom logic or handling of args then just execute +Then again you can just create another file `myseeder.go` and inside it do your custom logic or handling of args then +just execute `goseeder.Execute` Your `myseeder.go` might look like + ```go package main @@ -376,12 +401,14 @@ func main() { ``` -Then you have your server or app executable separate for example in `main.go` file, and the seeder functionality separated in `myseeder.go` +Then you have your server or app executable separate for example in `main.go` file, and the seeder functionality +separated in `myseeder.go` You can easily run your seeder `go run myseeder.go`, or build and run etc based on your requirements. You can pass all the necessary options to the `goSeeder.Execute` method. If you want to execute seeders for a particular env only (skip common seeds) for example you do it like: + ```go goseeder.Execute(con, goseeder.ForEnv("test"), goseeder.ShouldSkipCommon(true)) ``` @@ -389,19 +416,18 @@ goseeder.Execute(con, goseeder.ForEnv("test"), goseeder.ShouldSkipCommon(true)) These are the possible options you can give after the mandatory db connection: - `ForEnv(env string)` - you can specify here the env for which you want to execute - `ForSpecificSeeds(seedNames []string)` - just specify array of seed names you want to execute -- `ShouldSkipCommon(value bool)` - this option has effect only if also gsenv if set, then will not run the common seeds (seeds that do not have any env specified -- `ShouldSkipCommon(value bool)` - this option has effect only if also gsenv if set, then will not run the common seeds (seeds that do not have any env specified - - +- `ShouldSkipCommon(value bool)` - this option has effect only if also gsenv if set, then will not run the common + seeds (seeds that do not have any env specified ## Summary Of Cli Args -You can always run +You can always run + ```bash run go run main.go --help ```` -to see all the available arguments and their descriptions. +to see all the available arguments and their descriptions. For the current version the result is: @@ -424,12 +450,14 @@ Usage of /var/folders/rd/2bkszcpx6xgcddpn7f3bhczn1m9fb7/T/go-build358407825/b001 It is a special way of executing particular seeds always! in all environments, together with respective env seeds. If you want to skip common executions, the means to do so are provided already in this documentation. -From cli with the flag +From cli with the flag + ``` -gs-skip-common ``` and programmatically call `ShouldSkipCommon(true)`, like simple example: + ``` err := goseeder.Execute(con, goseeder.ForEnv("test"), goseeder.ShouldSkipCommon(true)) ``` diff --git a/examples/simpleshop/db/migrations/000001_init_schema.up.sql b/examples/simpleshop/db/migrations/000001_init_schema.up.sql index 8984f72..2fd8ce2 100644 --- a/examples/simpleshop/db/migrations/000001_init_schema.up.sql +++ b/examples/simpleshop/db/migrations/000001_init_schema.up.sql @@ -2,18 +2,18 @@ create schema if not exists goseeder_simpleshop collate utf8_general_ci; create table categories ( - id smallint unsigned auto_increment + id smallint unsigned auto_increment primary key, - name json not null, + name json not null, created_at timestamp default CURRENT_TIMESTAMP not null, updated_at timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP ); create table products ( - id char(36) not null + id char(36) not null primary key, - name json not null, + name json not null, created_at timestamp default CURRENT_TIMESTAMP not null, updated_at timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP ); diff --git a/helpers.go b/helpers.go index eb17419..2a03ec6 100644 --- a/helpers.go +++ b/helpers.go @@ -1,6 +1,7 @@ package goseeder import ( + "encoding/json" "fmt" "log" "reflect" @@ -92,6 +93,12 @@ func parseValue(value interface{}) interface{} { return value.(float64) case string: return value.(string) + case interface{}: + asJson, err := json.Marshal(value) + if err != nil { + panic(err) + } + return string(asJson) default: log.Printf("Don't know type : %v", v) } diff --git a/seeder.go b/seeder.go index f3e5c2c..528ceb2 100644 --- a/seeder.go +++ b/seeder.go @@ -83,7 +83,7 @@ func WithSeeder(conProvider func() *sql.DB, clientMain func()) { } } -// Register the given seed function as common to run for all environments +// Register the given seed function as common to run for all environments func Register(seeder func(s Seeder)) { RegisterForEnv("", seeder) } @@ -105,6 +105,15 @@ func RegisterForEnv(env string, seeder func(s Seeder)) { }) } +// RegisterForEnvNamed register the given seed function for a specific environment and give it a specific name +func RegisterForEnvNamed(env string, seeder func(s Seeder), name string) { + seeders = append(seeders, clientSeeder{ + env: env, + name: name, + cb: seeder, + }) +} + // Execute use this method for using this lib programmatically and executing // seeder directly with full flexibility. Be sure first to have registered your seeders func Execute(db *sql.DB, options ...ConfigOption) error { @@ -156,11 +165,11 @@ func Execute(db *sql.DB, options ...ConfigOption) error { func seed(seeder *Seeder) (rerr error) { clientSeeder := seeder.context start := time.Now() - printInfo(fmt.Sprintf("[%s] started seeding...\n", clientSeeder.name)) + printInfo(fmt.Sprintf("[%s][%s] started seeding...\n", clientSeeder.env, clientSeeder.name)) defer func() { if r := recover(); r != nil { - msg := fmt.Sprintf("[%s] seed failed: %+v\n", clientSeeder.name, r) + msg := fmt.Sprintf("[%s][%s] seed failed: %+v\n", clientSeeder.env, clientSeeder.name, r) printError(msg) rerr = errors.New(msg) } @@ -168,6 +177,6 @@ func seed(seeder *Seeder) (rerr error) { clientSeeder.cb(*seeder) elapsed := time.Since(start) - printInfo(fmt.Sprintf("[%s] seeded successfully, duration %s\n\n", clientSeeder.name, elapsed)) + printInfo(fmt.Sprintf("[%s][%s] seeded successfully, duration %s\n\n", clientSeeder.env, clientSeeder.name, elapsed)) return nil } diff --git a/seeder_test.go b/seeder_test.go index c1e4241..0cd7a17 100644 --- a/seeder_test.go +++ b/seeder_test.go @@ -158,4 +158,12 @@ func TestRegisterForEnv(t *testing.T) { require.NotEmpty(t, a.cb) } +func TestRegisterForEnvNamed(t *testing.T) { + RegisterForEnvNamed("mySuperEnv", dummySeeder, "customName") + a := seeders[len(seeders)-1] + require.Equal(t, "mySuperEnv", a.env) + require.Equal(t, "customName", a.name) + require.NotEmpty(t, a.cb) +} + func dummySeeder(s Seeder) {} diff --git a/sources.go b/sources.go index ef5f4e6..f7f7e67 100644 --- a/sources.go +++ b/sources.go @@ -17,6 +17,15 @@ func SetDataPath(path string) { //FromJson inserts into a database table with name same as the filename all the json entries func (s Seeder) FromJson(filename string) { + s.fromJson(filename, filename) +} + +//FromJsonIntoTable reads json from the given file and inserts into the given table name +func (s Seeder) FromJsonIntoTable(filename string, tableName string) { + s.fromJson(filename, tableName) +} + +func (s Seeder) fromJson(filename string, tableName string) { content, err := ioutil.ReadFile(fmt.Sprintf("%s/%s.json", dataPath, filename)) if err != nil { log.Fatal(err) @@ -29,7 +38,7 @@ func (s Seeder) FromJson(filename string) { } for _, e := range m { - stmQuery, args := prepareStatement(filename, e) + stmQuery, args := prepareStatement(tableName, e) stmt, _ := s.DB.Prepare(stmQuery.String()) _, err := stmt.Exec(args...) if err != nil {