diff --git a/go.mod b/go.mod index d14f967eda..4aa12abb27 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/projectdiscovery/gologger v1.0.0 github.com/projectdiscovery/retryabledns v1.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 - github.com/valyala/fasttemplate v1.1.0 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 28185ac420..a242a457f9 100644 --- a/go.sum +++ b/go.sum @@ -19,10 +19,6 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4= -github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/pkg/requests/dns-request.go b/pkg/requests/dns-request.go index d891a43a5d..7ba11d69d1 100644 --- a/pkg/requests/dns-request.go +++ b/pkg/requests/dns-request.go @@ -6,7 +6,6 @@ import ( "github.com/miekg/dns" "github.com/projectdiscovery/nuclei/pkg/extractors" "github.com/projectdiscovery/nuclei/pkg/matchers" - "github.com/valyala/fasttemplate" ) // DNSRequest contains a request to be made from a template @@ -17,6 +16,8 @@ type DNSRequest struct { Type string `yaml:"type"` Class string `yaml:"class"` Retries int `yaml:"retries"` + // Raw contains a raw request + Raw string `yaml:"raw,omitempty"` // Matchers contains the detection mechanism for the request to identify // whether the request was successful @@ -52,8 +53,9 @@ func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) { var q dns.Question - t := fasttemplate.New(r.Name, "{{", "}}") - q.Name = dns.Fqdn(t.ExecuteString(map[string]interface{}{"FQDN": domain})) + replacer := newReplacer(map[string]interface{}{"FQDN": domain}) + + q.Name = dns.Fqdn(replacer.Replace(r.Name)) q.Qclass = toQClass(r.Class) q.Qtype = toQType(r.Type) diff --git a/pkg/requests/http-request.go b/pkg/requests/http-request.go index c1d480b658..c986e07769 100644 --- a/pkg/requests/http-request.go +++ b/pkg/requests/http-request.go @@ -1,6 +1,8 @@ package requests import ( + "bufio" + "fmt" "io/ioutil" "net/http" "net/url" @@ -9,7 +11,6 @@ import ( "github.com/projectdiscovery/nuclei/pkg/extractors" "github.com/projectdiscovery/nuclei/pkg/matchers" retryablehttp "github.com/projectdiscovery/retryablehttp-go" - "github.com/valyala/fasttemplate" ) // HTTPRequest contains a request to be made from a template @@ -37,6 +38,8 @@ type HTTPRequest struct { Redirects bool `yaml:"redirects,omitempty"` // MaxRedirects is the maximum number of redirects that should be followed. MaxRedirects int `yaml:"max-redirects,omitempty"` + // Raw contains raw requests + Raw []string `yaml:"raw,omitempty"` } // GetMatchersCondition returns the condition for the matcher @@ -49,7 +52,7 @@ func (r *HTTPRequest) SetMatchersCondition(condition matchers.ConditionType) { r.matchersCondition = condition } -// MakeHTTPRequest creates a *http.Request from a request template +// MakeHTTPRequest creates a *http.Request from a request configuration func (r *HTTPRequest) MakeHTTPRequest(baseURL string) ([]*retryablehttp.Request, error) { parsed, err := url.Parse(baseURL) if err != nil { @@ -57,50 +60,71 @@ func (r *HTTPRequest) MakeHTTPRequest(baseURL string) ([]*retryablehttp.Request, } hostname := parsed.Hostname() - requests := make([]*retryablehttp.Request, 0, len(r.Path)) + values := map[string]interface{}{ + "BaseURL": baseURL, + "Hostname": hostname, + } + + if len(r.Raw) > 0 { + return r.makeHTTPRequestFromRaw(baseURL, values) + } + + return r.makeHTTPRequestFromModel(baseURL, values) +} + +// MakeHTTPRequestFromModel creates a *http.Request from a request template +func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests []*retryablehttp.Request, err error) { + replacer := newReplacer(values) for _, path := range r.Path { // Replace the dynamic variables in the URL if any - t := fasttemplate.New(path, "{{", "}}") - url := t.ExecuteString(map[string]interface{}{ - "BaseURL": baseURL, - "Hostname": hostname, - }) + URL := replacer.Replace(path) // Build a request on the specified URL - req, err := http.NewRequest(r.Method, url, nil) + req, err := http.NewRequest(r.Method, URL, nil) if err != nil { return nil, err } - // Check if the user requested a request body - if r.Body != "" { - req.Body = ioutil.NopCloser(strings.NewReader(r.Body)) + request, err := r.fillRequest(req, values) + if err != nil { + return nil, err } - // Set the header values requested - for header, value := range r.Headers { - t := fasttemplate.New(value, "{{", "}}") - val := t.ExecuteString(map[string]interface{}{ - "BaseURL": baseURL, - "Hostname": hostname, - }) - req.Header.Set(header, val) - } + requests = append(requests, request) + } - // Set some headers only if the header wasn't supplied by the user - if _, ok := r.Headers["User-Agent"]; !ok { - req.Header.Set("User-Agent", "Nuclei (@pdiscoveryio)") - } - if _, ok := r.Headers["Accept"]; !ok { - req.Header.Set("Accept", "*/*") + return +} + +// makeHTTPRequestFromRaw creates a *http.Request from a raw request +func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests []*retryablehttp.Request, err error) { + replacer := newReplacer(values) + for _, raw := range r.Raw { + // Add trailing line + raw += "\n" + + // Replace the dynamic variables in the URL if any + raw = replacer.Replace(raw) + + // Build a parsed request from raw + parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw))) + if err != nil { + return nil, err } - if _, ok := r.Headers["Accept-Language"]; !ok { - req.Header.Set("Accept-Language", "en") + + // requests generated from http.ReadRequest have incorrect RequestURI, so they + // cannot be used to perform another request directly, we need to generate a new one + // with the new target url + finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL) + req, err := http.NewRequest(r.Method, finalURL, parsedReq.Body) + if err != nil { + return nil, err } - req.Header.Set("Connection", "close") - req.Close = true - request, err := retryablehttp.FromRequest(req) + // copy headers + req.Header = parsedReq.Header.Clone() + + request, err := r.fillRequest(req, values) if err != nil { return nil, err } @@ -110,3 +134,32 @@ func (r *HTTPRequest) MakeHTTPRequest(baseURL string) ([]*retryablehttp.Request, return requests, nil } + +func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { + replacer := newReplacer(values) + // Check if the user requested a request body + if r.Body != "" { + req.Body = ioutil.NopCloser(strings.NewReader(r.Body)) + } + + // Set the header values requested + for header, value := range r.Headers { + req.Header.Set(header, replacer.Replace(value)) + } + + // Set some headers only if the header wasn't supplied by the user + if _, ok := req.Header["User-Agent"]; !ok { + req.Header.Set("User-Agent", "Nuclei (@pdiscoveryio)") + } + + if _, ok := req.Header["Accept"]; !ok { + req.Header.Set("Accept", "*/*") + } + if _, ok := req.Header["Accept-Language"]; !ok { + req.Header.Set("Accept-Language", "en") + } + req.Header.Set("Connection", "close") + req.Close = true + + return retryablehttp.FromRequest(req) +} diff --git a/pkg/requests/util.go b/pkg/requests/util.go new file mode 100644 index 0000000000..e7d466330c --- /dev/null +++ b/pkg/requests/util.go @@ -0,0 +1,16 @@ +package requests + +import ( + "fmt" + "strings" +) + +func newReplacer(values map[string]interface{}) *strings.Replacer { + var replacerItems []string + for k, v := range values { + replacerItems = append(replacerItems, fmt.Sprintf("{{%s}}", k)) + replacerItems = append(replacerItems, fmt.Sprintf("%s", v)) + } + + return strings.NewReplacer(replacerItems...) +}