Skip to content

Bot Fingerprint #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jeremy5189 merged 5 commits into from
Jun 5, 2025
Merged

Bot Fingerprint #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e25dccb
Add bot fingerprint in sha-inv page
jeremy5189 Jun 4, 2025
04dd755
Handle fingerprint in logs
jeremy5189 Jun 4, 2025
6403a79
Add full fingerprint json to log
jeremy5189 Jun 4, 2025
8fce760
Fine tune JS
jeremy5189 Jun 5, 2025
3297b77
Remove debug logs
jeremy5189 Jun 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 27 additions & 19 deletions internal/http_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,10 +347,14 @@ func accessGranted(
config *Config,
decisionListResultString string,
botScore float64,
botScoreTopFactor string) {
botScoreTopFactor string,
botFingerprint IntegrityCheckPayloadWrapper) {
if botScore >= 0 {
c.Header("X-Banjax-Bot-Score", fmt.Sprintf("%f", botScore))
c.Header("X-Banjax-Bot-Score-Top-Factor", botScoreTopFactor)
c.Header("X-Banjax-Bot-Fingerprint", botFingerprint.Hash)
jsonPayload, _ := json.Marshal(botFingerprint.Payload)
c.Header("X-Banjax-Bot-Fingerprint-Full", string(jsonPayload))
}
c.Header("X-Banjax-Decision", decisionListResultString)
c.Header("X-Accel-Redirect", "@access_granted") // nginx named location that proxy_passes to origin
Expand All @@ -363,10 +367,14 @@ func accessDenied(
config *Config,
decisionListResultString string,
botScore float64,
botScoreTopFactor string) {
botScoreTopFactor string,
botFingerprint IntegrityCheckPayloadWrapper) {
if botScore >= 0 {
c.Header("X-Banjax-Bot-Score", fmt.Sprintf("%f", botScore))
c.Header("X-Banjax-Bot-Score-Top-Factor", botScoreTopFactor)
c.Header("X-Banjax-Bot-Fingerprint", botFingerprint.Hash)
jsonPayload, _ := json.Marshal(botFingerprint.Payload)
c.Header("X-Banjax-Bot-Fingerprint-Full", string(jsonPayload))
}
c.Header("X-Banjax-Decision", decisionListResultString)
c.Header("Cache-Control", "no-cache,no-store") // XXX think about caching
Expand Down Expand Up @@ -572,7 +580,7 @@ func sendOrValidateShaChallenge(
clientUserAgent := c.Request.Header.Get("X-Client-User-Agent")
challengeCookie, err := c.Cookie(ChallengeCookieName)
integrityCheckCookie, _ := c.Cookie(IntegrityCheckCookieName)
botScore, botScoreTopFactor := integrityCheckCalcBotScoreWrapper(integrityCheckCookie)
botScore, botScoreTopFactor, botFingerprint := integrityCheckCalcBotScoreWrapper(integrityCheckCookie)
requestedMethod := c.Request.Method
if err == nil {
err := ValidateShaInvCookie(config.HmacSecret, challengeCookie, time.Now(), getUserAgentOrIp(c, config), config.ShaInvExpectedZeroBits)
Expand All @@ -581,7 +589,7 @@ func sendOrValidateShaChallenge(
// log.Println(err)
sendOrValidateShaChallengeResult.ShaChallengeResult = ShaChallengeFailedBadCookie
} else {
accessGranted(c, config, ShaChallengeResultToString[ShaChallengePassed], botScore, botScoreTopFactor)
accessGranted(c, config, ShaChallengeResultToString[ShaChallengePassed], botScore, botScoreTopFactor, botFingerprint)
ReportPassedFailedBannedMessage(config, "ip_passed_challenge", clientIp, requestedHost)
// log.Println("Sha-inverse challenge passed")
sendOrValidateShaChallengeResult.ShaChallengeResult = ShaChallengePassed
Expand All @@ -607,7 +615,7 @@ func sendOrValidateShaChallenge(
sendOrValidateShaChallengeResult.TooManyFailedChallengesResult = tooManyFailedChallengesResult
if tooManyFailedChallengesResult.Exceeded {
ReportPassedFailedBannedMessage(config, "ip_banned", clientIp, requestedHost)
accessDenied(c, config, "TooManyFailedChallenges", botScore, botScoreTopFactor)
accessDenied(c, config, "TooManyFailedChallenges", botScore, botScoreTopFactor, botFingerprint)
return sendOrValidateShaChallengeResult
}
}
Expand Down Expand Up @@ -690,7 +698,7 @@ func sendOrValidatePassword(
err := ValidatePasswordCookie(config.HmacSecret, passwordCookie, time.Now(), getUserAgentOrIp(c, config), expectedHashedPassword2)
if err == nil {
// roaming password passed, we do not record fail specifically for roaming fail
accessGranted(c, config, PasswordChallengeResultToString[PasswordChallengeRoamingPassed], -1.0, "")
accessGranted(c, config, PasswordChallengeResultToString[PasswordChallengeRoamingPassed], -1.0, "", IntegrityCheckPayloadWrapper{})
Copy link

Copilot AI Jun 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nitpick] Multiple calls create a default empty IntegrityCheckPayloadWrapper inline. Consider extracting this into a helper function or constant to reduce duplication and improve maintainability.

Suggested change
accessGranted(c, config, PasswordChallengeResultToString[PasswordChallengeRoamingPassed], -1.0, "", IntegrityCheckPayloadWrapper{})
accessGranted(c, config, PasswordChallengeResultToString[PasswordChallengeRoamingPassed], -1.0, "", DefaultIntegrityCheckPayloadWrapper())

Copilot uses AI. Check for mistakes.
ReportPassedFailedBannedMessage(config, "ip_passed_challenge", clientIp, requestedHost)
sendOrValidatePasswordResult.PasswordChallengeResult = PasswordChallengeRoamingPassed
return sendOrValidatePasswordResult
Expand All @@ -699,7 +707,7 @@ func sendOrValidatePassword(
sendOrValidatePasswordResult.PasswordChallengeResult = PasswordChallengeFailedBadCookie
}
} else {
accessGranted(c, config, PasswordChallengeResultToString[PasswordChallengePassed], -1.0, "")
accessGranted(c, config, PasswordChallengeResultToString[PasswordChallengePassed], -1.0, "", IntegrityCheckPayloadWrapper{})
ReportPassedFailedBannedMessage(config, "ip_passed_challenge", clientIp, requestedHost)
// log.Println("Password challenge passed")
sendOrValidatePasswordResult.PasswordChallengeResult = PasswordChallengePassed
Expand All @@ -725,7 +733,7 @@ func sendOrValidatePassword(
// log.Println(tooManyFailedChallengesResult)
if tooManyFailedChallengesResult.Exceeded {
ReportPassedFailedBannedMessage(config, "ip_banned", clientIp, requestedHost)
accessDenied(c, config, "TooManyFailedPassword", -1.0, "")
accessDenied(c, config, "TooManyFailedPassword", -1.0, "", IntegrityCheckPayloadWrapper{})
return sendOrValidatePasswordResult
}
_, allowRoaming := passwordProtectedPaths.GetExpandCookieDomain(requestedHost)
Expand Down Expand Up @@ -876,7 +884,7 @@ func decisionForNginx2(
}
if grantPriorityPass {
decisionForNginxResult.DecisionListResult = PasswordProtectedPriorityPass
accessGranted(c, config, DecisionListResultToString[PasswordProtectedPriorityPass], -1.0, "")
accessGranted(c, config, DecisionListResultToString[PasswordProtectedPriorityPass], -1.0, "", IntegrityCheckPayloadWrapper{})
return
}
}
Expand All @@ -898,7 +906,7 @@ func decisionForNginx2(
case PasswordProtectedException:
decisionForNginxResult.DecisionListResult = PasswordProtectedPathException
// FIXED: prevent password challenge exception path getting challenge
accessGranted(c, config, DecisionListResultToString[PasswordProtectedPathException], -1.0, "")
accessGranted(c, config, DecisionListResultToString[PasswordProtectedPathException], -1.0, "", IntegrityCheckPayloadWrapper{})
return
case NotPasswordProtected:
default:
Expand All @@ -912,7 +920,7 @@ func decisionForNginx2(
if foundInPerSiteList {
switch decision {
case Allow:
accessGranted(c, config, DecisionListResultToString[PerSiteAccessGranted], -1.0, "")
accessGranted(c, config, DecisionListResultToString[PerSiteAccessGranted], -1.0, "", IntegrityCheckPayloadWrapper{})
// log.Println("access granted from per-site lists")
decisionForNginxResult.DecisionListResult = PerSiteAccessGranted
return
Expand All @@ -931,7 +939,7 @@ func decisionForNginx2(
decisionForNginxResult.TooManyFailedChallengesResult = &sendOrValidateShaChallengeResult.TooManyFailedChallengesResult
return
case NginxBlock, IptablesBlock:
accessDenied(c, config, DecisionListResultToString[PerSiteBlock], -1.0, "")
accessDenied(c, config, DecisionListResultToString[PerSiteBlock], -1.0, "", IntegrityCheckPayloadWrapper{})
// log.Println("block from per-site lists")
decisionForNginxResult.DecisionListResult = PerSiteBlock
return
Expand All @@ -942,7 +950,7 @@ func decisionForNginx2(
if foundInGlobalList {
switch decision {
case Allow:
accessGranted(c, config, DecisionListResultToString[GlobalAccessGranted], -1.0, "")
accessGranted(c, config, DecisionListResultToString[GlobalAccessGranted], -1.0, "", IntegrityCheckPayloadWrapper{})
// log.Println("access granted from global lists")
decisionForNginxResult.DecisionListResult = GlobalAccessGranted
return
Expand All @@ -961,7 +969,7 @@ func decisionForNginx2(
decisionForNginxResult.TooManyFailedChallengesResult = &sendOrValidateShaChallengeResult.TooManyFailedChallengesResult
return
case NginxBlock, IptablesBlock:
accessDenied(c, config, DecisionListResultToString[GlobalBlock], -1.0, "")
accessDenied(c, config, DecisionListResultToString[GlobalBlock], -1.0, "", IntegrityCheckPayloadWrapper{})
// log.Println("access denied from global lists")
decisionForNginxResult.DecisionListResult = GlobalBlock
return
Expand All @@ -978,14 +986,14 @@ func decisionForNginx2(
} else {
switch expiringDecision.Decision {
case Allow:
accessGranted(c, config, DecisionListResultToString[ExpiringAccessGranted], -1.0, "")
accessGranted(c, config, DecisionListResultToString[ExpiringAccessGranted], -1.0, "", IntegrityCheckPayloadWrapper{})
// log.Println("access granted from expiring lists")
decisionForNginxResult.DecisionListResult = ExpiringAccessGranted
return
case Challenge:
// apply exception to both challenge from baskerville and regex banner
if checkPerSiteShaInvPathExceptions(config, requestedHost, requestedPath) {
accessGranted(c, config, DecisionListResultToString[PerSiteShaInvPathException], -1.0, "")
accessGranted(c, config, DecisionListResultToString[PerSiteShaInvPathException], -1.0, "", IntegrityCheckPayloadWrapper{})
decisionForNginxResult.DecisionListResult = PerSiteShaInvPathException
return
}
Expand All @@ -1009,7 +1017,7 @@ func decisionForNginx2(
return
}
case NginxBlock, IptablesBlock:
accessDenied(c, config, DecisionListResultToString[ExpiringBlock], -1.0, "")
accessDenied(c, config, DecisionListResultToString[ExpiringBlock], -1.0, "", IntegrityCheckPayloadWrapper{})
// log.Println("access denied from expiring lists")
decisionForNginxResult.DecisionListResult = ExpiringBlock
return
Expand All @@ -1026,7 +1034,7 @@ func decisionForNginx2(
// Reuse the exception from password prot for site-wide sha inv exceptions path
if passwordProtectedPaths.IsException(requestedHost, requestedProtectedPath) {
decisionForNginxResult.DecisionListResult = SiteWideChallengeException
accessGranted(c, config, DecisionListResultToString[SiteWideChallengeException], -1.0, "")
accessGranted(c, config, DecisionListResultToString[SiteWideChallengeException], -1.0, "", IntegrityCheckPayloadWrapper{})
} else {
sendOrValidateShaChallengeResult := sendOrValidateShaChallenge(
config,
Expand All @@ -1048,7 +1056,7 @@ func decisionForNginx2(
if decisionForNginxResult.DecisionListResult == NotSet {
decisionForNginxResult.DecisionListResult = NoMention
}
accessGranted(c, config, DecisionListResultToString[decisionForNginxResult.DecisionListResult], -1.0, "")
accessGranted(c, config, DecisionListResultToString[decisionForNginxResult.DecisionListResult], -1.0, "", IntegrityCheckPayloadWrapper{})
return
}

Expand Down
70 changes: 65 additions & 5 deletions internal/integrity_check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package internal

import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"math"
"strings"
Expand All @@ -28,10 +31,50 @@ type IntegrityCheckPayload struct {
Window IntegrityCheckWindowSize `json:"window"`
ColorDepth int `json:"colorDepth"`
LangLength int `json:"langLength"`
Language string `json:"language"`
Languages []string `json:"languages"`
Timezone string `json:"timezone"`
Platform string `json:"platform"`
CanvasFp string `json:"canvasFp"`
WebglFp string `json:"webglFp"`
MathFp string `json:"mathFp"`
Webcam bool `json:"webcam"`
}

type IntegrityCheckPayloadWrapper struct {
Payload IntegrityCheckPayload
Hash string
}

func integrityCheckCalcFingerprint(p IntegrityCheckPayload) string {
languages := strings.Join(p.Languages, ",")

raw := fmt.Sprintf(
"%s|%s|%s|%s|%d|%d|%d|%d|%dx%d|%s|%s|%s|%s|%t|%t|%t",
p.Platform,
p.Timezone,
p.Language,
languages,
p.CPU,
p.Memory,
p.ColorDepth,
p.LangLength,
p.Screen.Width, p.Screen.Height,
p.GPURenderer,
p.CanvasFp,
p.WebglFp,
p.MathFp,
p.Webdriver,
p.HasPlugins,
p.Webcam,
)

hash := sha256.Sum256([]byte(raw))
return hex.EncodeToString(hash[:])
}

// calculateBotScore calculates a bot score based on various payload properties
func integrityCheckCalcBotScore(p IntegrityCheckPayload) (float64, string) {
func integrityCheckCalcBotScore(p IntegrityCheckPayload) (float64, string, IntegrityCheckPayloadWrapper) {
factorWeights := map[string]int{
"webdriver": 10,
"no_plugins": 3,
Expand Down Expand Up @@ -121,17 +164,34 @@ func integrityCheckCalcBotScore(p IntegrityCheckPayload) (float64, string) {
}

normalized := math.Min(float64(score)/float64(maxScore), 1.0)
return normalized, topFactor
fingerprint := integrityCheckCalcFingerprint(p)
// log.Printf("Calculated bot score: %.2f, top factor: %s, fingerprint: %s", normalized, topFactor, fingerprint)

// Create a wrapper to return the payload and its hash
payloadWrapper := IntegrityCheckPayloadWrapper{
Payload: p,
Hash: fingerprint,
}

return normalized, topFactor, payloadWrapper
}

func integrityCheckCalcBotScoreWrapper(base64Payload string) (float64, string) {
func integrityCheckCalcBotScoreWrapper(base64Payload string) (float64, string, IntegrityCheckPayloadWrapper) {
// check if base64Payload is empty
if base64Payload == "" {
return 1.0, "no_payload" // return a high score if payload is empty
// return a high score if payload is empty
return 1.0, "no_payload", IntegrityCheckPayloadWrapper{
Payload: IntegrityCheckPayload{},
Hash: "",
}
}
payload, err := integrityCheckDecodePayload(base64Payload)
if err != nil {
return 1.0, "err_payload" // return a high score if payload is invalid
// return a high score if payload is invalid
return 1.0, "err_payload", IntegrityCheckPayloadWrapper{
Payload: IntegrityCheckPayload{},
Hash: "",
}
}
return integrityCheckCalcBotScore(payload)
}
Expand Down
14 changes: 12 additions & 2 deletions internal/integrity_check_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,16 @@ func TestCalculateBotScore(t *testing.T) {

for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
score, factor := integrityCheckCalcBotScore(c.payload)
score, factor, fingerprint := integrityCheckCalcBotScore(c.payload)
if score != c.expected {
t.Errorf("Test %s failed: expected score %.2f, got %.2f", c.name, c.expected, score)
}
if factor != c.expectedFactor {
t.Errorf("Test %s failed: expected factor %s, got %s", c.name, c.expectedFactor, factor)
}
if fingerprint.Hash == "" {
t.Errorf("Test %s failed: expected non-empty fingerprint, got empty", c.name)
}
})
}
}
Expand All @@ -80,36 +83,43 @@ func TestCalculateBotScoreWrapper(t *testing.T) {
base64Input string
expectedScore float64
expectedFactor string
expectedFingerprint string
}{
{
name: "Valid payload",
base64Input: encoded,
expectedScore: 1.0,
expectedFactor: "webdriver",
expectedFingerprint: integrityCheckCalcFingerprint(payload),
},
{
name: "Invalid base64",
base64Input: "not a base64 string",
expectedScore: 1.0,
expectedFactor: "err_payload",
expectedFingerprint: "",
},
{
name: "No payload",
base64Input: "",
expectedScore: 1.0,
expectedFactor: "no_payload",
expectedFingerprint: "",
},
}

for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
score, factor := integrityCheckCalcBotScoreWrapper(c.base64Input)
score, factor, fingerprint := integrityCheckCalcBotScoreWrapper(c.base64Input)
if score != c.expectedScore {
t.Errorf("Test %s failed: expected score %.2f, got %.2f", c.name, c.expectedScore, score)
}
if factor != c.expectedFactor {
t.Errorf("Test %s failed: expected factor %s, got %s", c.name, c.expectedFactor, factor)
}
if fingerprint.Hash != c.expectedFingerprint {
t.Errorf("Test %s failed: expected fingerprint %s, got %s", c.name, c.expectedFingerprint, fingerprint.Hash)
}
})
}
}
Loading