From 20f5b15825ba10ab20c4d01d277721fed649013f Mon Sep 17 00:00:00 2001 From: soerlemans Date: Sun, 1 Dec 2024 12:36:13 +0100 Subject: [PATCH 1/4] Implemented rate limit for shortscan cmd. --- pkg/shortscan/shortscan.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkg/shortscan/shortscan.go b/pkg/shortscan/shortscan.go index 798ef35..9627fbf 100644 --- a/pkg/shortscan/shortscan.go +++ b/pkg/shortscan/shortscan.go @@ -136,6 +136,9 @@ var statusCache map[string]map[int]struct{} var distanceCache map[string]map[int]distances var checksumRegex *regexp.Regexp +// Delay between requests set by the rate limit. +var requestDelay time.Duration + // Command-line arguments and help type arguments struct { Urls []string `arg:"positional,required" help:"url to scan (multiple URLs can be provided; a file containing URLs can be specified with an «at» prefix, for example: @urls.txt)" placeholder:"URL"` @@ -148,6 +151,7 @@ type arguments struct { FullUrl bool `arg:"-F" help:"display the full URL for confirmed files rather than just the filename" default:"false"` NoRecurse bool `arg:"-n" help:"don't detect and recurse into subdirectories (disabled when autocomplete is disabled)" default:"false"` Stabilise bool `arg:"-s" help:"attempt to get coherent autocomplete results from an unstable server (generates more requests)" default:"false"` + Rate float64 `arg:"-r" help:"maximum requests per second" default:"1.0"` Patience int `arg:"-p" help:"patience level when determining vulnerability (0 = patient; 1 = very patient)" placeholder:"LEVEL" default:"0"` Characters string `arg:"-C" help:"filename characters to enumerate" default:"JFKGOTMYVHSPCANDXLRWEBQUIZ8549176320-_()&'!#$%@^{}~"` Autocomplete string `arg:"-a" help:"autocomplete detection mode (auto = autoselect; method = HTTP method magic; status = HTTP status; distance = Levenshtein distance; none = disable)" placeholder:"mode" default:"auto"` @@ -170,8 +174,25 @@ func pathEscape(url string) string { return strings.Replace(nurl.QueryEscape(url), "+", "%20", -1) } +// Helper variable to the requestDelay function, keeps track of the time since the last request +var lastRequestTime time.Time + +// delayRequest delays a request if it were to go over the rate limit +func delayRequest() { + // Sleep to prevent going over the rate limit + sleepDuration := time.Until(lastRequestTime.Add(requestDelay)) + time.Sleep(sleepDuration) + + log.WithFields(log.Fields{"lastRequestTime": lastRequestTime, "requestDelay": requestDelay}).Info("Rate") + + // Update last request time + lastRequestTime = time.Now() +} + // fetch requests the given URL and returns an HTTP response object, handling retries gracefully func fetch(hc *http.Client, st *httpStats, method string, url string) (*http.Response, error) { + // Sleep until we are within rate limit constraints + delayRequest() // Create a request object req, err := http.NewRequest(method, url, nil) @@ -1047,6 +1068,14 @@ func Run() { p.Fail("output must be one of: human, json") } + // If the rate limit is a negative value we should error + if args.Rate < 0 { + p.Fail("negative rate limit is not allowed") + } + + // Initialize the calculated delay between requests, according to rate limit + requestDelay = time.Duration(1_000_000/args.Rate) * time.Microsecond + // Build the list of URLs to scan var urls []string for _, url := range args.Urls { From afd70911833dc9d85300e9eca992536a61b2a782 Mon Sep 17 00:00:00 2001 From: soerlemans Date: Sun, 1 Dec 2024 12:57:54 +0100 Subject: [PATCH 2/4] Changed rate limit value from float64 to uint. --- pkg/shortscan/shortscan.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/shortscan/shortscan.go b/pkg/shortscan/shortscan.go index 9627fbf..375d1c5 100644 --- a/pkg/shortscan/shortscan.go +++ b/pkg/shortscan/shortscan.go @@ -151,7 +151,7 @@ type arguments struct { FullUrl bool `arg:"-F" help:"display the full URL for confirmed files rather than just the filename" default:"false"` NoRecurse bool `arg:"-n" help:"don't detect and recurse into subdirectories (disabled when autocomplete is disabled)" default:"false"` Stabilise bool `arg:"-s" help:"attempt to get coherent autocomplete results from an unstable server (generates more requests)" default:"false"` - Rate float64 `arg:"-r" help:"maximum requests per second" default:"1.0"` + Rate uint `arg:"-r" help:"maximum requests per second" default:"0"` Patience int `arg:"-p" help:"patience level when determining vulnerability (0 = patient; 1 = very patient)" placeholder:"LEVEL" default:"0"` Characters string `arg:"-C" help:"filename characters to enumerate" default:"JFKGOTMYVHSPCANDXLRWEBQUIZ8549176320-_()&'!#$%@^{}~"` Autocomplete string `arg:"-a" help:"autocomplete detection mode (auto = autoselect; method = HTTP method magic; status = HTTP status; distance = Levenshtein distance; none = disable)" placeholder:"mode" default:"auto"` @@ -1068,13 +1068,14 @@ func Run() { p.Fail("output must be one of: human, json") } - // If the rate limit is a negative value we should error - if args.Rate < 0 { - p.Fail("negative rate limit is not allowed") + // If the rate limit is a negative treat it as no rate limit + delay := args.Rate + if args.Rate != 0 { + delay = 1_000_000/args.Rate } // Initialize the calculated delay between requests, according to rate limit - requestDelay = time.Duration(1_000_000/args.Rate) * time.Microsecond + requestDelay = time.Duration(delay) * time.Microsecond // Build the list of URLs to scan var urls []string From 9554fef47117f60f09eebbe10dbb449a6dea1948 Mon Sep 17 00:00:00 2001 From: soerlemans Date: Sun, 1 Dec 2024 13:01:44 +0100 Subject: [PATCH 3/4] Corrected logging info message. --- pkg/shortscan/shortscan.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/shortscan/shortscan.go b/pkg/shortscan/shortscan.go index 375d1c5..afe3152 100644 --- a/pkg/shortscan/shortscan.go +++ b/pkg/shortscan/shortscan.go @@ -183,7 +183,7 @@ func delayRequest() { sleepDuration := time.Until(lastRequestTime.Add(requestDelay)) time.Sleep(sleepDuration) - log.WithFields(log.Fields{"lastRequestTime": lastRequestTime, "requestDelay": requestDelay}).Info("Rate") + log.WithFields(log.Fields{"lastRequestTime": lastRequestTime, "requestDelay": requestDelay}).Info("Rate limit metadata") // Update last request time lastRequestTime = time.Now() @@ -1071,7 +1071,7 @@ func Run() { // If the rate limit is a negative treat it as no rate limit delay := args.Rate if args.Rate != 0 { - delay = 1_000_000/args.Rate + delay = 1_000_000 / args.Rate } // Initialize the calculated delay between requests, according to rate limit From 21eb0876db4924e9b98755a3e67b80edb06bb5ed Mon Sep 17 00:00:00 2001 From: soerlemans Date: Fri, 28 Feb 2025 20:46:39 +0100 Subject: [PATCH 4/4] Added support for fractal rate limit. --- pkg/shortscan/shortscan.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/shortscan/shortscan.go b/pkg/shortscan/shortscan.go index afe3152..2bf5bd4 100644 --- a/pkg/shortscan/shortscan.go +++ b/pkg/shortscan/shortscan.go @@ -151,7 +151,7 @@ type arguments struct { FullUrl bool `arg:"-F" help:"display the full URL for confirmed files rather than just the filename" default:"false"` NoRecurse bool `arg:"-n" help:"don't detect and recurse into subdirectories (disabled when autocomplete is disabled)" default:"false"` Stabilise bool `arg:"-s" help:"attempt to get coherent autocomplete results from an unstable server (generates more requests)" default:"false"` - Rate uint `arg:"-r" help:"maximum requests per second" default:"0"` + Rate float32 `arg:"-r" help:"maximum requests per second, supports floating point values" default:"0.0"` Patience int `arg:"-p" help:"patience level when determining vulnerability (0 = patient; 1 = very patient)" placeholder:"LEVEL" default:"0"` Characters string `arg:"-C" help:"filename characters to enumerate" default:"JFKGOTMYVHSPCANDXLRWEBQUIZ8549176320-_()&'!#$%@^{}~"` Autocomplete string `arg:"-a" help:"autocomplete detection mode (auto = autoselect; method = HTTP method magic; status = HTTP status; distance = Levenshtein distance; none = disable)" placeholder:"mode" default:"auto"` @@ -1071,7 +1071,7 @@ func Run() { // If the rate limit is a negative treat it as no rate limit delay := args.Rate if args.Rate != 0 { - delay = 1_000_000 / args.Rate + delay = 1_000_000.0 / args.Rate } // Initialize the calculated delay between requests, according to rate limit