Refactor the http.request from the js part of k6 to lib.netext.httpext (

grafana#928)

I would've liked if I have moved more or ... less but this is the amount
of code that will be useful for future development and is the amount
that I could move with some amount of certainty that I am not totally
breaking the whole project.

There is the very real possibility that some types like URL and Response
might need to be copied because of further problems with goja but for
the moment this appears to be working.

js/common/bridge.go

@@ -44,7 +44,13 @@ var (
 	)
 )
 
-// Returns the JS name for an exported struct field. The name is snake_cased, with respect for
+// if a fieldName is the key of this map exactly than the value for the given key should be used as
+// the name of the field in js
+var fieldNameExceptions = map[string]string{
+	"OCSP": "ocsp",
+}
+
+// FieldName Returns the JS name for an exported struct field. The name is snake_cased, with respect for
 // certain common initialisms (URL, ID, HTTP, etc).
 func FieldName(t reflect.Type, f reflect.StructField) string {
 	// PkgPath is non-empty for unexported fields.
@@ -61,18 +67,24 @@ func FieldName(t reflect.Type, f reflect.StructField) string {
 		return tag
 	}
 
+	if exception, ok := fieldNameExceptions[f.Name]; ok {
+		return exception
+	}
+
 	// Default to lowercasing the first character of the field name.
 	return snaker.CamelToSnake(f.Name)
 }
 
 // if a methodName is the key of this map exactly than the value for the given key should be used as
 // the name of the method in js
-var methodNameExceptions map[string]string = map[string]string{
+var methodNameExceptions = map[string]string{
 	"JSON": "json",
 	"HTML": "html",
+	"URL":  "url",
+	"OCSP": "ocsp",
 }
 
-// Returns the JS name for an exported method. The first letter of the method's name is
+// MethodName Returns the JS name for an exported method. The first letter of the method's name is
 // lowercased, otherwise it is unaltered.
 func MethodName(t reflect.Type, m reflect.Method) string {
 	// A field with a name beginning with an X is a constructor, and just gets the prefix stripped.
@@ -91,11 +103,15 @@ func MethodName(t reflect.Type, m reflect.Method) string {
 // FieldNameMapper for goja.Runtime.SetFieldNameMapper()
 type FieldNameMapper struct{}
 
+// FieldName is part of the goja.FieldNameMapper interface
+// https://godoc.org/github.com/dop251/goja#FieldNameMapper
 func (FieldNameMapper) FieldName(t reflect.Type, f reflect.StructField) string { return FieldName(t, f) }
 
+// MethodName is part of the goja.FieldNameMapper interface
+// https://godoc.org/github.com/dop251/goja#FieldNameMapper
 func (FieldNameMapper) MethodName(t reflect.Type, m reflect.Method) string { return MethodName(t, m) }
 
-// Binds an object's members to the global scope. Returns a function that un-binds them.
+// BindToGlobal Binds an object's members to the global scope. Returns a function that un-binds them.
 // Note that this will panic if passed something that isn't a struct; please don't do that.
 func BindToGlobal(rt *goja.Runtime, data map[string]interface{}) func() {
 	keys := make([]string, len(data))
@@ -113,6 +129,7 @@ func BindToGlobal(rt *goja.Runtime, data map[string]interface{}) func() {
 	}
 }
 
+// Bind the provided value v to the provided runtime
 func Bind(rt *goja.Runtime, v interface{}, ctxPtr *context.Context) map[string]interface{} {
 	exports := make(map[string]interface{})
 

js/common/context.go

@@ -29,22 +29,9 @@ import (
 type ctxKey int
 
 const (
-	ctxKeyState ctxKey = iota
-	ctxKeyRuntime
+	ctxKeyRuntime ctxKey = iota
 )
 
-func WithState(ctx context.Context, state *State) context.Context {
-	return context.WithValue(ctx, ctxKeyState, state)
-}
-
-func GetState(ctx context.Context) *State {
-	v := ctx.Value(ctxKeyState)
-	if v == nil {
-		return nil
-	}
-	return v.(*State)
-}
-
 func WithRuntime(ctx context.Context, rt *goja.Runtime) context.Context {
 	return context.WithValue(ctx, ctxKeyRuntime, rt)
 }

js/common/context_test.go

@@ -28,15 +28,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestContextState(t *testing.T) {
-	st := &State{}
-	assert.Equal(t, st, GetState(WithState(context.Background(), st)))
-}
-
-func TestContextStateNil(t *testing.T) {
-	assert.Nil(t, GetState(context.Background()))
-}
-
 func TestContextRuntime(t *testing.T) {
 	rt := goja.New()
 	assert.Equal(t, rt, GetRuntime(WithRuntime(context.Background(), rt)))

js/initcontext_test.go

@@ -390,7 +390,7 @@ func TestRequestWithBinaryFile(t *testing.T) {
 	logger.Level = log.DebugLevel
 	logger.Out = ioutil.Discard
 
-	state := &common.State{
+	state := &lib.State{
 		Options: lib.Options{},
 		Logger:  logger,
 		Group:   root,
@@ -406,7 +406,7 @@ func TestRequestWithBinaryFile(t *testing.T) {
 	}
 
 	ctx := context.Background()
-	ctx = common.WithState(ctx, state)
+	ctx = lib.WithState(ctx, state)
 	ctx = common.WithRuntime(ctx, bi.Runtime)
 	*bi.Context = ctx
 

js/modules/k6/crypto/crypto_test.go

@@ -184,10 +184,10 @@ func TestStreamingApi(t *testing.T) {
 	rt.SetFieldNameMapper(common.FieldNameMapper{})
 
 	root, _ := lib.NewGroup("", nil)
-	state := &common.State{Group: root}
+	state := &lib.State{Group: root}
 
 	ctx := context.Background()
-	ctx = common.WithState(ctx, state)
+	ctx = lib.WithState(ctx, state)
 	ctx = common.WithRuntime(ctx, rt)
 
 	rt.Set("crypto", common.Bind(rt, New(), &ctx))
@@ -249,10 +249,10 @@ func TestOutputEncoding(t *testing.T) {
 	rt.SetFieldNameMapper(common.FieldNameMapper{})
 
 	root, _ := lib.NewGroup("", nil)
-	state := &common.State{Group: root}
+	state := &lib.State{Group: root}
 
 	ctx := context.Background()
-	ctx = common.WithState(ctx, state)
+	ctx = lib.WithState(ctx, state)
 	ctx = common.WithRuntime(ctx, rt)
 
 	rt.Set("crypto", common.Bind(rt, New(), &ctx))
@@ -310,10 +310,10 @@ func TestHMac(t *testing.T) {
 	rt.SetFieldNameMapper(common.FieldNameMapper{})
 
 	root, _ := lib.NewGroup("", nil)
-	state := &common.State{Group: root}
+	state := &lib.State{Group: root}
 
 	ctx := context.Background()
-	ctx = common.WithState(ctx, state)
+	ctx = lib.WithState(ctx, state)
 	ctx = common.WithRuntime(ctx, rt)
 
 	rt.Set("crypto", common.Bind(rt, New(), &ctx))

js/modules/k6/http/cookiejar.go

@@ -33,6 +33,7 @@ import (
 	"github.com/pkg/errors"
 )
 
+// HTTPCookieJar is cookiejar.Jar wrapper to be used in js scripts
 type HTTPCookieJar struct {
 	jar *cookiejar.Jar
 	ctx *context.Context
@@ -46,6 +47,7 @@ func newCookieJar(ctxPtr *context.Context) *HTTPCookieJar {
 	return &HTTPCookieJar{jar, ctxPtr}
 }
 
+// CookiesForURL return the cookies for a given url as a map of key and values
 func (j HTTPCookieJar) CookiesForURL(url string) map[string][]string {
 	u, err := neturl.Parse(url)
 	if err != nil {
@@ -60,6 +62,7 @@ func (j HTTPCookieJar) CookiesForURL(url string) map[string][]string {
 	return objs
 }
 
+// Set sets a cookie for a particular url with the given name value and additional opts
 func (j HTTPCookieJar) Set(url, name, value string, opts goja.Value) (bool, error) {
 	rt := common.GetRuntime(*j.ctx)
 
@@ -74,10 +77,6 @@ func (j HTTPCookieJar) Set(url, name, value string, opts goja.Value) (bool, erro
 		params := paramsV.ToObject(rt)
 		for _, k := range params.Keys() {
 			switch strings.ToLower(k) {
-			case "name":
-				c.Name = params.Get(k).String()
-			case "value":
-				c.Value = params.Get(k).String()
 			case "path":
 				c.Path = params.Get(k).String()
 			case "domain":

js/modules/k6/http/http.go

@@ -22,15 +22,10 @@ package http
 
 import (
 	"context"
-	"net/http"
-	"net/http/cookiejar"
-
-	"fmt"
-	"net/http/httputil"
 
 	"github.com/loadimpact/k6/js/common"
+	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/netext"
-	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -46,18 +41,6 @@ const (
 // ErrJarForbiddenInInitContext is used when a cookie jar was made in the init context
 var ErrJarForbiddenInInitContext = common.NewInitContextError("Making cookie jars in the init context is not supported")
 
-type HTTPCookie struct {
-	Name, Value, Domain, Path string
-	HttpOnly, Secure          bool
-	MaxAge                    int
-	Expires                   int64
-}
-
-type HTTPRequestCookie struct {
-	Name, Value string
-	Replace     bool
-}
-
 type HTTP struct {
 	SSL_3_0                            string `js:"SSL_3_0"`
 	TLS_1_0                            string `js:"TLS_1_0"`
@@ -108,56 +91,9 @@ func (*HTTP) XCookieJar(ctx *context.Context) *HTTPCookieJar {
 }
 
 func (*HTTP) CookieJar(ctx context.Context) (*HTTPCookieJar, error) {
-	state := common.GetState(ctx)
+	state := lib.GetState(ctx)
 	if state == nil {
 		return nil, ErrJarForbiddenInInitContext
 	}
 	return &HTTPCookieJar{state.CookieJar, &ctx}, nil
 }
-
-func (*HTTP) mergeCookies(req *http.Request, jar *cookiejar.Jar, reqCookies map[string]*HTTPRequestCookie) map[string][]*HTTPRequestCookie {
-	allCookies := make(map[string][]*HTTPRequestCookie)
-	for _, c := range jar.Cookies(req.URL) {
-		allCookies[c.Name] = append(allCookies[c.Name], &HTTPRequestCookie{Name: c.Name, Value: c.Value})
-	}
-	for key, reqCookie := range reqCookies {
-		if jc := allCookies[key]; jc != nil && reqCookie.Replace {
-			allCookies[key] = []*HTTPRequestCookie{{Name: key, Value: reqCookie.Value}}
-		} else {
-			allCookies[key] = append(allCookies[key], &HTTPRequestCookie{Name: key, Value: reqCookie.Value})
-		}
-	}
-	return allCookies
-}
-
-func (*HTTP) setRequestCookies(req *http.Request, reqCookies map[string][]*HTTPRequestCookie) {
-	for _, cookies := range reqCookies {
-		for _, c := range cookies {
-			req.AddCookie(&http.Cookie{Name: c.Name, Value: c.Value})
-		}
-	}
-}
-
-func (*HTTP) debugRequest(state *common.State, req *http.Request, description string) {
-	if state.Options.HttpDebug.String != "" {
-		dump, err := httputil.DumpRequestOut(req, state.Options.HttpDebug.String == "full")
-		if err != nil {
-			log.Fatal(err)
-		}
-		logDump(description, dump)
-	}
-}
-
-func (*HTTP) debugResponse(state *common.State, res *http.Response, description string) {
-	if state.Options.HttpDebug.String != "" && res != nil {
-		dump, err := httputil.DumpResponse(res, state.Options.HttpDebug.String == "full")
-		if err != nil {
-			log.Fatal(err)
-		}
-		logDump(description, dump)
-	}
-}
-
-func logDump(description string, dump []byte) {
-	fmt.Printf("%s:\n%s\n", description, dump)
-}

js/modules/k6/http/http_test.go

@@ -21,12 +21,13 @@
 package http
 
 import (
-	"net/url"
 	"testing"
 
 	"github.com/dop251/goja"
 	"github.com/loadimpact/k6/js/common"
+	"github.com/loadimpact/k6/lib/netext/httpext"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestTagURL(t *testing.T) {
@@ -43,9 +44,8 @@ func TestTagURL(t *testing.T) {
 	}
 	for expr, data := range testdata {
 		t.Run("expr="+expr, func(t *testing.T) {
-			u, err := url.Parse(data.u)
-			assert.NoError(t, err)
-			tag := URL{u, data.n, data.u}
+			tag, err := httpext.NewURL(data.u, data.n)
+			require.NoError(t, err)
 			v, err := common.RunString(rt, "http.url`"+expr+"`")
 			if assert.NoError(t, err) {
 				assert.Equal(t, tag, v.Export())

js/modules/k6/http/http_url.go

@@ -22,37 +22,30 @@ package http
 
 import (
 	"fmt"
-	"net/url"
 
 	"github.com/dop251/goja"
+	"github.com/loadimpact/k6/lib/netext/httpext"
 )
 
-// A URL wraps net.URL, and preserves the template (if any) the URL was constructed from.
-type URL struct {
-	URL       *url.URL `js:"-"`
-	Name      string   `js:"name"` // http://example.com/thing/${}/
-	URLString string   `js:"url"`  // http://example.com/thing/1234/
-}
-
 // ToURL tries to convert anything passed to it to a k6 URL struct
-func ToURL(u interface{}) (URL, error) {
+func ToURL(u interface{}) (httpext.URL, error) {
 	switch tu := u.(type) {
-	case URL:
+	case httpext.URL:
 		// Handling of http.url`http://example.com/{$id}`
 		return tu, nil
 	case string:
 		// Handling of "http://example.com/"
-		u, err := url.Parse(tu)
-		return URL{u, tu, tu}, err
+		return httpext.NewURL(tu, tu)
 	case goja.Value:
 		// Unwrap goja values
 		return ToURL(tu.Export())
 	default:
-		return URL{}, fmt.Errorf("invalid URL value '%#v'", u)
+		return httpext.URL{}, fmt.Errorf("invalid URL value '%#v'", u)
 	}
 }
 
-func (http *HTTP) Url(parts []string, pieces ...string) (URL, error) {
+// URL creates new URL from the provided parts
+func (http *HTTP) URL(parts []string, pieces ...string) (httpext.URL, error) {
 	var name, urlstr string
 	for i, part := range parts {
 		name += part
@@ -62,6 +55,5 @@ func (http *HTTP) Url(parts []string, pieces ...string) (URL, error) {
 			urlstr += pieces[i]
 		}
 	}
-	u, err := url.Parse(urlstr)
-	return URL{u, name, urlstr}, err
+	return httpext.NewURL(urlstr, name)
 }