Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: call SetupStream and SetupMetrics from nginx configuration #105

Merged
merged 6 commits into from
Apr 3, 2025
Merged

feat: call SetupStream and SetupMetrics from nginx configuration #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d70256d
feat: remove stream mode and metrics setup
sabban Mar 27, 2025
2af02f4
feat(lib): add get_mode function to retrieve runtime mode
sabban Mar 27, 2025
561be3d
feat(crowdsec): expose SetupMetrics and SetupStream functions
sabban Mar 27, 2025
b6d5439
fix: correct function reference in timer setup
sabban Mar 27, 2025
971c0a2
feat: add init_worker_by_lua_block for stream mode and metrics initia…
sabban Mar 31, 2025
58bfd18
fix: correct debug cache order in test file
sabban Mar 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 7 additions & 10 deletions lib/crowdsec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ function csmod.debug_metrics()
ngx.log(ngx.DEBUG, "Shortening metrics period to 15 seconds")
end

function csmod.get_mode()
return runtime.conf["MODE"]
end

--- init function
-- init function called by nginx in init_by_lua_block
-- @param configFile path to the configuration file
Expand Down Expand Up @@ -166,7 +170,7 @@ end
--- The idea here is to setup the timer that will trigger the metrics sending
--- If first run then just fire the new timer to run the function again in METRICS_PERIOD
--- If not send metrics and run the timer again in METRICS_PERIOD
local function Setup_metrics()
function csmod.SetupMetrics()
-- if no API_URL, we don't setup metrics
if runtime.conf["API_URL"] == "" then
return
Expand All @@ -177,7 +181,7 @@ local function Setup_metrics()
ngx.log(ngx.INFO, "worker is exiting, not setting up metrics timer")
return
end
local ok, err = ngx.timer.at(METRICS_PERIOD, Setup_metrics)
local ok, err = ngx.timer.at(METRICS_PERIOD, csmod.SetupMetrics)
if not ok then
error("Failed to create the timer: " .. (err or "unknown"))
else
Expand Down Expand Up @@ -252,7 +256,7 @@ function csmod.GetCaptchaBackendKey()
return captcha.GetCaptchaBackendKey()
end

local function SetupStream()
function csmod.SetupStream()
local function SetupStreamTimer()
if ngx.worker.exiting() then
ngx.log(ngx.INFO, "worker is exiting, not setting up stream timer")
Expand Down Expand Up @@ -336,11 +340,6 @@ function csmod.allowIp(ip)
return true, nil, nil
end

if runtime.conf["MODE"] == "stream" then
ngx.log(ngx.DEBUG, "stream mode")
SetupStream()
end

local key, ip_version = utils.item_to_string(ip, "ip")
if key == nil then
return true, nil, "Check failed '" .. ip .. "' has no valid IP address"
Expand Down Expand Up @@ -548,8 +547,6 @@ function csmod.Allow(ip)
end
end

Setup_metrics()

local ok, remediation, err = csmod.allowIp(ip)
if err ~= nil then
ngx.log(ngx.ERR, "[Crowdsec] bouncer error: " .. err)
Expand Down
15 changes: 15 additions & 0 deletions t/01lua_config.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,21 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
end
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

--- config
location = /t {
content_by_lua_block {
Expand Down
16 changes: 15 additions & 1 deletion t/05stream_and_ban.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;

Expand Down Expand Up @@ -118,8 +132,8 @@ DEBUG CACHE:metrics_processed/ip_type=ipv4&:1
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,
DEBUG CACHE:captcha_ok:false
DEBUG CACHE:first_run:true
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:startup:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_processed/ip_type=ipv4&:2
DEBUG CACHE:decision_cache/ipv4_4294967295_16843009:ban/CAPI/ipv4
Expand Down
14 changes: 14 additions & 0 deletions t/06stream_and_not_ban.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;

Expand Down
20 changes: 17 additions & 3 deletions t/07stream_expiration_and_cache.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;

Expand Down Expand Up @@ -142,19 +156,19 @@ DEBUG CACHE:metrics_processed/ip_type=ipv4&:1
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,
DEBUG CACHE:captcha_ok:false
DEBUG CACHE:first_run:true
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:startup:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_processed/ip_type=ipv4&:2
DEBUG CACHE:decision_cache/ipv4_4294967295_16843009:ban/CAPI/ipv4
DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:1
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI&,
DEBUG CACHE:captcha_ok:false
DEBUG CACHE:first_run:true
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:1
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:startup:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:1
DEBUG CACHE:metrics_processed/ip_type=ipv4&:3
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI&,
DEBUG CACHE:captcha_ok:false
20 changes: 16 additions & 4 deletions t/08_stream_ban_and_cache.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;

Expand Down Expand Up @@ -140,22 +154,20 @@ DEBUG CACHE:metrics_processed/ip_type=ipv4&:1
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,
DEBUG CACHE:captcha_ok:false
DEBUG CACHE:first_run:true
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:startup:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_processed/ip_type=ipv4&:2
DEBUG CACHE:decision_cache/ipv4_4294967295_16843009:ban/CAPI/ipv4
DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:1
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI&,
DEBUG CACHE:captcha_ok:false
DEBUG CACHE:first_run:true
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:startup:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_processed/ip_type=ipv4&:3
DEBUG CACHE:decision_cache/ipv4_4294967295_16843009:ban/CAPI/ipv4
DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:2
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI&,
DEBUG CACHE:captcha_ok:false


14 changes: 14 additions & 0 deletions t/09_stream_ban_and_metrics.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;

Expand Down
14 changes: 14 additions & 0 deletions t/10_live_ban_and_cache.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;
location = /v1/decisions {
Expand Down
14 changes: 14 additions & 0 deletions t/11_live_ban_and_metrics.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;

Expand Down
18 changes: 16 additions & 2 deletions t/12_stream_ban_and_deletion.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;

Expand Down Expand Up @@ -140,19 +154,19 @@ DEBUG CACHE:metrics_processed/ip_type=ipv4&:1
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,
DEBUG CACHE:captcha_ok:false
DEBUG CACHE:first_run:true
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:startup:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_processed/ip_type=ipv4&:2
DEBUG CACHE:decision_cache/ipv4_4294967295_16843009:ban/CAPI/ipv4
DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:1
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI&,
DEBUG CACHE:captcha_ok:false
DEBUG CACHE:first_run:true
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:1
DEBUG CACHE:decision_cache/ipv4_4294967295_16843010:ban/CAPI/ipv4
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_processed/ip_type=ipv4&:3
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI&,
Expand Down
20 changes: 17 additions & 3 deletions t/13_stream_captcha_and_deletion.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ run_tests();

__DATA__

=== TEST 5: Stream mode block test
=== TEST 13: Stream mode block test

--- init

Expand Down Expand Up @@ -94,6 +94,20 @@ access_by_lua_block {
cs.Allow(ngx.var.remote_addr)
}

init_worker_by_lua_block {
cs = require "crowdsec"
local mode = cs.get_mode()
if string.lower(mode) == "stream" then
ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id()))
cs.SetupStream()
end

if ngx.worker.id() == 0 then
ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id()))
cs.SetupMetrics()
end
}

server {
listen 8081;

Expand Down Expand Up @@ -154,8 +168,8 @@ DEBUG CACHE:metrics_processed/ip_type=ipv4&:1
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,
DEBUG CACHE:captcha_ok:true
DEBUG CACHE:first_run:true
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:startup:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_processed/ip_type=ipv4&:2
DEBUG CACHE:decision_cache/ipv4_4294967295_16843009:captcha/CAPI/ipv4
Expand All @@ -164,10 +178,10 @@ DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI
DEBUG CACHE:captcha_ok:true
DEBUG CACHE:captcha_1.1.1.1:/t
DEBUG CACHE:first_run:true
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:1
DEBUG CACHE:decision_cache/ipv4_4294967295_16843010:ban/CAPI/ipv4
DEBUG CACHE:startup:false
DEBUG CACHE:metrics_first_run:false
DEBUG CACHE:refreshing:false
DEBUG CACHE:metrics_processed/ip_type=ipv4&:3
DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI&,
Expand Down
Loading