From a1d10461a3fb86e7a63020dbdaf91a5bb089cbef Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Fri, 11 Apr 2025 18:09:44 +0300 Subject: [PATCH 1/3] Handle loadbalancing for multiple web hosts --- lib/kamal/cli/main.rb | 10 ++ lib/kamal/cli/proxy.rb | 112 +++++++++++++++++- lib/kamal/commander.rb | 8 ++ lib/kamal/commands/loadbalancer.rb | 100 ++++++++++++++++ lib/kamal/commands/proxy.rb | 4 + lib/kamal/configuration.rb | 4 + lib/kamal/configuration/docs/proxy.yml | 7 ++ lib/kamal/configuration/loadbalancer.rb | 28 +++++ lib/kamal/configuration/proxy.rb | 26 +++- lib/kamal/configuration/role.rb | 3 +- lib/kamal/configuration/validator/proxy.rb | 4 +- test/cli/cli_test_case.rb | 3 + test/configuration/proxy_test.rb | 59 +++++++++ .../docker/deployer/app/config/deploy.yml | 1 + .../deployer/app_with_roles/config/deploy.yml | 1 + 15 files changed, 364 insertions(+), 6 deletions(-) create mode 100644 lib/kamal/commands/loadbalancer.rb create mode 100644 lib/kamal/configuration/loadbalancer.rb diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 4af6d8788..b147b6e96 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -41,6 +41,11 @@ def deploy(boot_accessories: false) invoke "kamal:cli:app:boot", [], invoke_options + if KAMAL.config.proxy.load_balancing? + say "Updating loadbalancer configuration...", :magenta + invoke "kamal:cli:proxy:loadbalancer", [ "deploy" ], invoke_options + end + say "Prune old containers and images...", :magenta invoke "kamal:cli:prune:all", [], invoke_options end @@ -70,6 +75,11 @@ def redeploy invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true) invoke "kamal:cli:app:boot", [], invoke_options + + if KAMAL.config.proxy.load_balancing? + say "Updating loadbalancer configuration...", :magenta + invoke "kamal:cli:proxy:loadbalancer", [ "deploy" ], invoke_options + end end end diff --git a/lib/kamal/cli/proxy.rb b/lib/kamal/cli/proxy.rb index 0e00af6c8..6b4ab9993 100644 --- a/lib/kamal/cli/proxy.rb +++ b/lib/kamal/cli/proxy.rb @@ -19,6 +19,14 @@ def boot execute *KAMAL.proxy.ensure_apps_config_directory execute *KAMAL.proxy.start_or_run end + + if KAMAL.config.proxy.load_balancing? + on(KAMAL.config.proxy.effective_loadbalancer) do |host| + info "Starting loadbalancer on #{host}..." + execute *KAMAL.registry.login + execute *KAMAL.loadbalancer.start_or_run + end + end end end @@ -114,7 +122,7 @@ def reboot execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug execute *KAMAL.registry.login - "Stopping and removing kamal-proxy on #{host}, if running..." + info "Stopping and removing kamal-proxy on #{host}, if running..." execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false execute *KAMAL.proxy.remove_container execute *KAMAL.proxy.ensure_apps_config_directory @@ -123,6 +131,24 @@ def reboot end run_hook "post-proxy-reboot", hosts: host_list end + + if KAMAL.config.proxy.load_balancing? + lb_host = KAMAL.config.proxy.effective_loadbalancer + run_hook "pre-loadbalancer-reboot", hosts: lb_host + + on(lb_host) do |host| + execute *KAMAL.auditor.record("Rebooted loadbalancer"), verbosity: :debug + execute *KAMAL.registry.login + + info "Stopping and removing load-balancer on #{host}, if running..." + execute *KAMAL.loadbalancer.stop, raise_on_non_zero_exit: false + execute *KAMAL.loadbalancer.remove_container + + execute *KAMAL.loadbalancer.run + end + + run_hook "post-loadbalancer-reboot", hosts: lb_host + end end end end @@ -143,10 +169,10 @@ def upgrade execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug execute *KAMAL.registry.login - "Stopping and removing Traefik on #{host}, if running..." + info "Stopping and removing Traefik on #{host}, if running..." execute *KAMAL.proxy.cleanup_traefik - "Stopping and removing kamal-proxy on #{host}, if running..." + info "Stopping and removing kamal-proxy on #{host}, if running..." execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false execute *KAMAL.proxy.remove_container execute *KAMAL.proxy.remove_image @@ -198,6 +224,12 @@ def restart desc "details", "Show details about proxy container from servers" def details on(KAMAL.proxy_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.proxy.info), type: "Proxy" } + + if KAMAL.config.proxy.load_balancing? + on(KAMAL.config.proxy.effective_loadbalancer) do |host| + puts_by_host host, capture_with_info(*KAMAL.proxy.loadbalancer.info), type: "Loadbalancer" + end + end end desc "logs", "Show log lines from proxy on servers" @@ -239,6 +271,66 @@ def remove end end + desc "loadbalancer STATUS", "Manage the load balancer" + def loadbalancer(status) + case status + when "info" + if KAMAL.config.proxy.load_balancing? + on(KAMAL.config.proxy.effective_loadbalancer) do |host| + puts "Loadbalancer status on #{host}:" + puts capture_with_info(*KAMAL.loadbalancer.info) + end + else + puts "Load balancing is not configured" + end + when "start" + if KAMAL.config.proxy.load_balancing? + on(KAMAL.config.proxy.effective_loadbalancer) do |host| + execute *KAMAL.registry.login + execute *KAMAL.loadbalancer.start_or_run + end + else + puts "Load balancing is not configured" + end + when "stop" + if KAMAL.config.proxy.load_balancing? + on(KAMAL.config.proxy.effective_loadbalancer) do |host| + execute *KAMAL.loadbalancer.stop, raise_on_non_zero_exit: false + end + else + puts "Load balancing is not configured" + end + when "logs" + if KAMAL.config.proxy.load_balancing? + on(KAMAL.config.proxy.effective_loadbalancer) do |host| + puts_by_host host, capture(*KAMAL.loadbalancer.logs(timestamps: true)), type: "Loadbalancer" + end + else + puts "Load balancing is not configured" + end + when "deploy" + if KAMAL.config.proxy.load_balancing? + targets = [] + KAMAL.config.roles.each do |role| + next unless role.running_proxy? + + role.hosts.each do |host| + targets << host + end + end + + on(KAMAL.config.proxy.effective_loadbalancer) do |host| + info "Deploying to loadbalancer on #{host} with targets: #{targets.join(', ')}" + execute *KAMAL.loadbalancer.deploy(targets: targets) + end + else + puts "Load balancing is not configured" + end + else + puts "Unknown loadbalancer subcommand: #{status}. Available: info, start, stop, logs, deploy" + end + end + desc "remove_container", "Remove proxy container from servers", hide: true def remove_container with_lock do @@ -246,6 +338,13 @@ def remove_container execute *KAMAL.auditor.record("Removed proxy container"), verbosity: :debug execute *KAMAL.proxy.remove_container end + + if KAMAL.config.proxy.load_balancing? + on(KAMAL.config.proxy.effective_loadbalancer) do + execute *KAMAL.auditor.record("Removed loadbalancer container"), verbosity: :debug + execute *KAMAL.loadbalancer.remove_container + end + end end end @@ -256,6 +355,13 @@ def remove_image execute *KAMAL.auditor.record("Removed proxy image"), verbosity: :debug execute *KAMAL.proxy.remove_image end + + if KAMAL.config.proxy.load_balancing? + on(KAMAL.config.proxy.effective_loadbalancer) do + execute *KAMAL.auditor.record("Removed loadbalancer image"), verbosity: :debug + execute *KAMAL.loadbalancer.remove_image + end + end end end diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index 0882311d5..bdf86d3b8 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -113,6 +113,14 @@ def proxy @commands[:proxy] ||= Kamal::Commands::Proxy.new(config) end + def loadbalancer_config + @loadbalancer_config ||= Kamal::Configuration::Loadbalancer.new(config: config, proxy_config: config.proxy.proxy_config) + end + + def loadbalancer + @commands[:loadbalancer] ||= Kamal::Commands::Loadbalancer.new(config, loadbalancer_config: loadbalancer_config) + end + def prune @commands[:prune] ||= Kamal::Commands::Prune.new(config) end diff --git a/lib/kamal/commands/loadbalancer.rb b/lib/kamal/commands/loadbalancer.rb new file mode 100644 index 000000000..0a05c2843 --- /dev/null +++ b/lib/kamal/commands/loadbalancer.rb @@ -0,0 +1,100 @@ +class Kamal::Commands::Loadbalancer < Kamal::Commands::Base + delegate :argumentize, :optionize, to: Kamal::Utils + + attr_reader :loadbalancer_config + + def initialize(config, loadbalancer_config: nil) + super(config) + @loadbalancer_config = loadbalancer_config + end + + def run + pipe \ + [ :echo, proxy_image ], + xargs(docker(:run, + "--name", container_name, + "--network", "kamal", + "--detach", + "--restart", "unless-stopped", + "--publish", "80:80", + "--publish", "443:443", + "--label", "org.opencontainers.image.title=kamal-loadbalancer", + "--volume", "kamal-loadbalancer-config:/home/kamal-loadbalancer/.config/kamal-loadbalancer")) + end + + def start + docker :container, :start, container_name + end + + def stop(name: container_name) + docker :container, :stop, name + end + + def start_or_run + combine start, run, by: "||" + end + + def deploy(targets: []) + target_args = targets.map { |t| "#{t}:80" } + + hosts = loadbalancer_config.hosts + + options = [] + options << "--target=#{target_args.join(',')}" + options << "--host=#{hosts.join(',')}" + options << "--tls" if loadbalancer_config.ssl? + + docker :exec, container_name, "kamal-proxy", "deploy", loadbalancer_config.config.service, *options + end + + def info + docker :ps, "--filter", "name=^#{container_name}$" + end + + def version + pipe \ + docker(:inspect, container_name, "--format '{{.Config.Image}}'"), + [ :cut, "-d:", "-f2" ] + end + + def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil) + pipe \ + docker(:logs, container_name, ("--since #{since}" if since), ("--tail #{lines}" if lines), ("--timestamps" if timestamps), "2>&1"), + ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep) + end + + def follow_logs(host:, timestamps: true, grep: nil, grep_options: nil) + run_over_ssh pipe( + docker(:logs, container_name, ("--timestamps" if timestamps), "--tail", "10", "--follow", "2>&1"), + (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep) + ).join(" "), host: host + end + + def remove_container + docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=kamal-loadbalancer" + end + + def remove_image + docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=kamal-loadbalancer" + end + + def ensure_directory + make_directory loadbalancer_config.directory + end + + def remove_directory + super(loadbalancer_config.directory) + end + + private + def proxy_image + [ + loadbalancer_config.config.proxy_boot.image_default, + Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION + ].join(":") + end + + def container_name + loadbalancer_config.container_name + end +end diff --git a/lib/kamal/commands/proxy.rb b/lib/kamal/commands/proxy.rb index 7699dde24..ee190ea34 100644 --- a/lib/kamal/commands/proxy.rb +++ b/lib/kamal/commands/proxy.rb @@ -105,6 +105,10 @@ def reset_run_command remove_file config.proxy_boot.run_command_file end + def loadbalancer + @loadbalancer ||= Kamal::Commands::Loadbalancer.new(config, loadbalancer_config: KAMAL.loadbalancer_config) + end + private def container_name config.proxy_boot.container_name diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 6435fa57c..b2de0c6d0 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -145,6 +145,10 @@ def proxy_roles roles.select(&:running_proxy?) end + def load_balancing? + proxy&.load_balancing? + end + def proxy_role_names proxy_roles.flat_map(&:name) end diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index a127bfea0..5d2f8b1d3 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -25,6 +25,13 @@ proxy: hosts: - foo.example.com - bar.example.com + + # Loadbalancer + # + # Specify a host to run the loadbalancer on. The loadbalancer will distribute requests + # to all web hosts. If not specified but multiple web hosts are configured, the first + # web host will be used as the loadbalancer host. + loadbalancer: lb.example.com # App port # diff --git a/lib/kamal/configuration/loadbalancer.rb b/lib/kamal/configuration/loadbalancer.rb new file mode 100644 index 000000000..fe847ce69 --- /dev/null +++ b/lib/kamal/configuration/loadbalancer.rb @@ -0,0 +1,28 @@ +class Kamal::Configuration::Loadbalancer < Kamal::Configuration::Proxy + CONTAINER_NAME = "load-balancer".freeze + + def self.validation_config_key + "proxy" + end + + def initialize(config:, proxy_config:) + super(config: config, proxy_config: proxy_config) + end + + def deploy_options + opts = super + + opts[:host] = hosts if hosts.present? + opts[:tls] = proxy_config["ssl"].presence + + opts + end + + def directory + File.join config.run_directory, "loadbalancer" + end + + def container_name + CONTAINER_NAME + end +end diff --git a/lib/kamal/configuration/proxy.rb b/lib/kamal/configuration/proxy.rb index ccb4ac423..86d770a71 100644 --- a/lib/kamal/configuration/proxy.rb +++ b/lib/kamal/configuration/proxy.rb @@ -3,6 +3,7 @@ class Kamal::Configuration::Proxy DEFAULT_LOG_REQUEST_HEADERS = [ "Cache-Control", "Last-Modified", "User-Agent" ] CONTAINER_NAME = "kamal-proxy" + LOADBALANCER_CONTAINER_NAME = "kamal-loadbalancer" delegate :argumentize, :optionize, to: Kamal::Utils @@ -27,8 +28,24 @@ def hosts proxy_config["hosts"] || proxy_config["host"]&.split(",") || [] end + def loadbalancer + proxy_config["loadbalancer"] + end + + def load_balancing? + effective_loadbalancer.present? + end + + def effective_loadbalancer + return false if loadbalancer == false + return loadbalancer if loadbalancer.present? + return config.primary_role.hosts.first if config.primary_role && Array(config.primary_role.hosts).size > 1 + + nil + end + def deploy_options - { + opts = { host: hosts, tls: proxy_config["ssl"].presence, "deploy-timeout": seconds_duration(config.deploy_timeout), @@ -48,6 +65,13 @@ def deploy_options "log-response-header": proxy_config.dig("logging", "response_headers"), "error-pages": error_pages }.compact + + if load_balancing? + opts.delete(:host) + opts.delete(:tls) + end + + opts end def deploy_command_args(target:) diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb index c6bd8783c..5c164130b 100644 --- a/lib/kamal/configuration/role.rb +++ b/lib/kamal/configuration/role.rb @@ -150,7 +150,8 @@ def asset_volume_directory(version = config.version) end def ensure_one_host_for_ssl - if running_proxy? && proxy.ssl? && hosts.size > 1 + # Skip SSL validation when a loadbalancer is present, as the loadbalancer handles SSL termination + if running_proxy? && proxy.ssl? && hosts.size > 1 && !proxy.loadbalancer.present? raise Kamal::ConfigurationError, "SSL is only supported on a single server, found #{hosts.size} servers for role #{name}" end end diff --git a/lib/kamal/configuration/validator/proxy.rb b/lib/kamal/configuration/validator/proxy.rb index b9e11cd99..c1127ef2b 100644 --- a/lib/kamal/configuration/validator/proxy.rb +++ b/lib/kamal/configuration/validator/proxy.rb @@ -3,7 +3,9 @@ def validate! unless config.nil? super - if config["host"].blank? && config["hosts"].blank? && config["ssl"] + # Skip SSL host validation when a loadbalancer is present + # since SSL is disabled when using a loadbalancer + if config["host"].blank? && config["hosts"].blank? && config["ssl"] && config["loadbalancer"].blank? error "Must set a host to enable automatic SSL" end diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index b7ca9ecaf..6ea337cfc 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -7,6 +7,9 @@ class CliTestCase < ActiveSupport::TestCase ENV["MYSQL_ROOT_PASSWORD"] = "secret123" Object.send(:remove_const, :KAMAL) Object.const_set(:KAMAL, Kamal::Commander.new) + + # Ensure no loadbalancer functionality interferes with tests + Kamal::Configuration::Proxy.any_instance.stubs(:load_balancing?).returns(false) end teardown do diff --git a/test/configuration/proxy_test.rb b/test/configuration/proxy_test.rb index 460d30cb5..d0904e2c0 100644 --- a/test/configuration/proxy_test.rb +++ b/test/configuration/proxy_test.rb @@ -38,6 +38,65 @@ class ConfigurationProxyTest < ActiveSupport::TestCase assert_not config.proxy.ssl? end + test "loadbalancer present" do + @deploy[:proxy] = { "loadbalancer" => "lb.example.com" } + assert_equal "lb.example.com", config.proxy.loadbalancer + end + + test "loadbalancer not present" do + @deploy[:proxy] = {} + assert_nil config.proxy.loadbalancer + end + + test "effective_loadbalancer with explicit loadbalancer" do + @deploy[:proxy] = { "loadbalancer" => "lb.example.com" } + assert_equal "lb.example.com", config.proxy.effective_loadbalancer + end + + test "effective_loadbalancer with multiple web hosts but no explicit loadbalancer" do + @deploy[:proxy] = { "hosts" => [] } + @deploy[:servers] = { "web" => [ "web1.example.com", "web2.example.com" ] } + assert_equal "web1.example.com", config.proxy.effective_loadbalancer + end + + test "effective_loadbalancer with single web host and no explicit loadbalancer" do + @deploy[:proxy] = { "hosts" => [] } + @deploy[:servers] = { "web" => [ "web1.example.com" ] } + assert_nil config.proxy.effective_loadbalancer + end + + test "load_balancing? returns true when loadbalancer is present" do + @deploy[:proxy] = { "loadbalancer" => "lb.example.com" } + assert config.proxy.load_balancing? + end + + test "load_balancing? returns true when multiple web hosts are present" do + @deploy[:proxy] = { "hosts" => [] } + @deploy[:servers] = { "web" => [ "web1.example.com", "web2.example.com" ] } + assert config.proxy.load_balancing? + end + + test "load_balancing? returns false when no loadbalancer and single web host" do + @deploy[:proxy] = { "hosts" => [] } + @deploy[:servers] = { "web" => [ "web1.example.com" ] } + assert_not config.proxy.load_balancing? + end + + test "deploy_options disables SSL when loadbalancer is present" do + @deploy[:proxy] = { "loadbalancer" => "lb.example.com" } + assert_equal false, config.proxy.deploy_options.key?(:tls) + end + + test "deploy_options makes hosts optional when loadbalancer is present" do + @deploy[:proxy] = { "loadbalancer" => "lb.example.com", "hosts" => [ "app1.example.com" ] } + assert_nil config.proxy.deploy_options[:host] + end + + test "deploy_options uses hosts when no loadbalancer is present" do + @deploy[:proxy] = { "hosts" => [ "app1.example.com" ] } + assert_equal [ "app1.example.com" ], config.proxy.deploy_options[:host] + end + test "false not allowed" do @deploy[:proxy] = false assert_raises(Kamal::ConfigurationError, "proxy: should be a hash") do diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index c0a576a49..2dffa50ec 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -25,6 +25,7 @@ drain_timeout: 2 readiness_delay: 0 proxy: host: 127.0.0.1 + loadbalancer: false registry: server: registry:4443 username: root diff --git a/test/integration/docker/deployer/app_with_roles/config/deploy.yml b/test/integration/docker/deployer/app_with_roles/config/deploy.yml index 6c5ef9f4f..39185de58 100644 --- a/test/integration/docker/deployer/app_with_roles/config/deploy.yml +++ b/test/integration/docker/deployer/app_with_roles/config/deploy.yml @@ -16,6 +16,7 @@ readiness_delay: 0 proxy: host: localhost ssl: false + loadbalancer: false healthcheck: interval: 1 timeout: 1 From a0b828e70ce2cf4d2891bd1396975a8484bcf837 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Sun, 5 Oct 2025 18:07:34 +0200 Subject: [PATCH 2/3] Revert #1656 --- lib/kamal/cli/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kamal/cli/build.rb b/lib/kamal/cli/build.rb index 9a76e5f2c..2327943e7 100644 --- a/lib/kamal/cli/build.rb +++ b/lib/kamal/cli/build.rb @@ -19,7 +19,7 @@ def push pre_connect_if_required ensure_docker_installed - login_to_registry_locally if KAMAL.builder.login_to_registry_locally? + login_to_registry_locally run_hook "pre-build" From 7ba6846ca29fe692d50515a5c0c3199e795a5e27 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Thu, 16 Oct 2025 11:03:20 +0200 Subject: [PATCH 3/3] Fix initializer to accept and provide secrets --- lib/kamal/commander.rb | 2 +- lib/kamal/configuration/loadbalancer.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index bdf86d3b8..a274d90d9 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -114,7 +114,7 @@ def proxy end def loadbalancer_config - @loadbalancer_config ||= Kamal::Configuration::Loadbalancer.new(config: config, proxy_config: config.proxy.proxy_config) + @loadbalancer_config ||= Kamal::Configuration::Loadbalancer.new(config: config, proxy_config: config.proxy.proxy_config, secrets: config.secrets) end def loadbalancer diff --git a/lib/kamal/configuration/loadbalancer.rb b/lib/kamal/configuration/loadbalancer.rb index fe847ce69..4794fbfe3 100644 --- a/lib/kamal/configuration/loadbalancer.rb +++ b/lib/kamal/configuration/loadbalancer.rb @@ -5,8 +5,8 @@ def self.validation_config_key "proxy" end - def initialize(config:, proxy_config:) - super(config: config, proxy_config: proxy_config) + def initialize(config:, proxy_config:, secrets:) + super end def deploy_options