Skip to content
This repository was archived by the owner on Dec 19, 2019. It is now read-only.

Multi-process support, drain commands, etc... #35

Open
gshively11 wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Multi-process support, drain commands, etc... #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
37d180c
Added support for sockets and nbproc > 1. Also added a drain command
gshively11 Jun 14, 2016
69c4fce
Removed state restrictions on the enable/disable/drain commands, they…
gshively11 Jun 14, 2016
4b210ee
Added a reload_kill_zombies command to deal with bugs in haproxy relo…
gshively11 Feb 13, 2017
143ca85
Forgot to increment seconds_waited in the reload_kill_zombies command…
gshively11 Feb 28, 2017
614ed63
Verifying reload command succeeded before taking further action in th…
gshively11 Jun 14, 2017
8624a48
Added support for global server-state-file. Commands that stop/restar…
gshively11 Jun 14, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 74 additions & 29 deletions bin/haproxyctl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,20 @@ begin
start
end
when 'reload'
if pidof
if pidof
reload(pidof)
else
puts 'haproxy not running. starting...'
start
end
when /reload_kill_zombies\s(\d+)/
if pidof
seconds_to_wait = Regexp.last_match[1].to_i
reload_kill_zombies(pidof, seconds_to_wait)
else
puts 'haproxy not running. starting...'
start
end
when 'status'
if pidof
puts "haproxy is running on pid(s) #{pidof.join(', ')}.\nthese ports are used and guys are connected:"
Expand Down Expand Up @@ -101,56 +109,62 @@ begin
# else
# puts 'status err haproxy is not running!'
# end
when 'show health'
status = unixsock('show stat')
when /(\d\s)?show health/
process = Regexp.last_match[1].to_i
status = unixsock('show stat', process)
status.each do |line|
data = line.split(',')
printf "%-30s %-30s %-7s %3s\n", data[0], data[1], data[17], data[18]
end
when /show backend(s?)/
status = unixsock('show stat').grep(/BACKEND/)
when /(\d\s)?show backends?/
process = Regexp.last_match[1].to_i
status = unixsock('show stat', process).grep(/BACKEND/)
status.each do |line|
data = line.split(',')
printf "%-30s %-30s %-7s %3s\n", data[0], data[1], data[17], data[18]
end
when /disable all EXCEPT (.+)/
servername = Regexp.last_match[ 1]
status = unixsock('show stat')
when /(\d\s)?disable all EXCEPT (.+)/
process = Regexp.last_match[1].to_i
servername = Regexp.last_match[2]
status = unixsock('show stat', process)
backend = status.grep(/#{servername}/)
backend.each do |line|
backend_group = line.split(',')
status.each do |pool|
data = pool.split(',')
if (data[0] == backend_group[0]) && ( data[1] !~ /#{servername}|BACKEND|FRONTEND/) && ( data[17] == 'UP')
unixsock("disable server #{data[0]}/#{data[1]}")
if (data[0] == backend_group[0]) && ( data[1] !~ /#{servername}|BACKEND|FRONTEND/)
unixsock("disable server #{data[0]}/#{data[1]}", process)
end
end
end
when /disable all (.+)/
servername = Regexp.last_match[ 1]
status = unixsock('show stat')
when /(\d\s)?disable all (.+)/
process = Regexp.last_match[1].to_i
servername = Regexp.last_match[2]
status = unixsock('show stat', process)
status.each do |line|
data = line.split(',')
if ( data[1] == servername) && ( data[17] == 'UP')
unixsock("disable server #{data[0]}/#{servername}")
if ( data[1] == servername)
unixsock("disable server #{data[0]}/#{servername}", process)
end
end
when /enable all EXCEPT (.+)/
servername = Regexp.last_match[ 1]
status = unixsock('show stat')
when /(\d\s)?enable all EXCEPT (.+)/
process = Regexp.last_match[1].to_i
servername = Regexp.last_match[2]
status = unixsock('show stat', process)
backend = status.grep(/#{servername}/)
backend.each do |line|
backend_group = line.split(',')
status.each do |pool|
data = pool.split(',')
if (data[0] == backend_group[0]) && ( data[1] !~ /#{servername}|BACKEND|FRONTEND/) && ( data[17] =~ /Down|MAINT/i)
unixsock("enable server #{data[0]}/#{data[1]}")
if (data[0] == backend_group[0]) && ( data[1] !~ /#{servername}|BACKEND|FRONTEND/)
unixsock("enable server #{data[0]}/#{data[1]}", process)
end
end
end
when /show stat (.+)/
fieldnames = Regexp.last_match[ 1]
status = unixsock('show stat')
when /(\d\s)?show stat (.+)/
process = Regexp.last_match[1].to_i
fieldnames = Regexp.last_match[2]
status = unixsock('show stat', process)
indices = fieldnames.split(' ').map do |name|
status.first.split(',').index(name) || begin
$stderr.puts("no such field: #{name}")
Expand All @@ -163,19 +177,50 @@ begin
filtered = indices.map { |index| row[index] }
puts (row[0...2] + filtered).compact.join(',')
end
when /enable all (.+)/
servername = Regexp.last_match[ 1]
status = unixsock('show stat')
when /(\d\s)?enable all (.+)/
process = Regexp.last_match[1].to_i
servername = Regexp.last_match[2]
status = unixsock('show stat', process)
status.each do |line|
data = line.split(',')
if ( data[1] == servername) && ( data[17] =~ /Down|MAINT/i)
unixsock("enable server #{data[0]}/#{servername}")
if ( data[1] == servername)
unixsock("enable server #{data[0]}/#{servername}", process)
end
end
when /(\d\s)?drain all (.+)/
process = Regexp.last_match[1].to_i
servername = Regexp.last_match[2]
status = unixsock('show stat', process)
status.each do |line|
data = line.split(',')
if ( data[1] == servername)
unixsock("set server #{data[0]}/#{servername} state drain", process)
end
end
when /(\d\s)?drain all EXCEPT (.+)/
process = Regexp.last_match[1].to_i
servername = Regexp.last_match[2]
status = unixsock('show stat', process)
backend = status.grep(/#{servername}/)
backend.each do |line|
backend_group = line.split(',')
status.each do |pool|
data = pool.split(',')
if (data[0] == backend_group[0]) && ( data[1] !~ /#{servername}|BACKEND|FRONTEND/)
unixsock("set server #{data[0]}/#{data[1]} state drain", process)
end
end
end
when 'version'
version
when /(\d\s)?(.*)/
process = Regexp.last_match[1].to_i
command = Regexp.last_match[2]
puts unixsock(command, process)
else
puts unixsock(argument)
# this case shouldn't be reached due to the
# above regex, but leaving it here just in case
puts unixsock(argument, nil)
end
rescue Errno::ENOENT => e
STDERR.puts e
Expand Down
162 changes: 117 additions & 45 deletions lib/haproxyctl.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ def start

def stop(pids)
if pids
write_server_state_file
puts "stopping haproxy on pids #{pids.join(', ')}..."
pids.each { |pid| system("kill #{pid}") || system("kill -9 #{pid}") }
puts '... stopped'
Expand All @@ -29,34 +30,100 @@ def stop(pids)
end

def reload(pids)
if pids
puts "gracefully stopping connections on pids #{pids.join(', ')}..."
system("#{exec} -D -f #{config_path} -p #{pidfile} -sf $(cat #{pidfile})")
puts "checking if connections still alive on #{pids.join(', ')}..."
nowpids = check_running
while pids == nowpids
puts "still haven't killed old pids.
waiting 2s for existing connections to die...
(ctrl+c to stop this check)"
unless pids
puts 'haproxy is not running!'
return
end
write_server_state_file
pids_string = pids.join(' ')
puts "gracefully stopping connections on pids #{pids_string}..."
reload_succeeded = system("#{exec} -D -f #{config_path} -p #{pidfile} -sf $(cat #{pidfile})")
puts "checking if connections still alive on #{pids_string}..."
nowpids = check_running
if !reload_succeeded
puts "pids no longer exist, haproxy may no longer be running" unless pids == nowpids
puts "failed to reload haproxy, check errors above"
return
end
while pids == nowpids
puts "still haven't killed old pids.
waiting 2s for existing connections to die...
(ctrl+c to stop this check)"
sleep 2
nowpids = check_running || 0
end
puts "reloaded haproxy on pids #{nowpids.join(', ')}"
end

def reload_kill_zombies(pids, seconds_to_wait)
unless pids
puts 'haproxy is not running!'
return
end
write_server_state_file
pids_string = pids.join(' ')
puts "gracefully stopping connections on pids #{pids_string}..."
reload_succeeded = system("#{exec} -D -f #{config_path} -p #{pidfile} -sf #{pids_string}")
puts "checking if connections still alive on #{pids_string}..."
nowpids = check_running
if !reload_succeeded
puts "pids no longer exist, haproxy may no longer be running" unless pids == nowpids
puts "failed to reload haproxy, check errors above"
return
end
while pids == nowpids
puts "still haven't killed old pids.
waiting 2s for existing connections to die...
(ctrl+c to stop this check)"
sleep 2
nowpids = check_running || 0
end
puts "reloaded haproxy on pids #{nowpids.join(', ')}"
puts "ensuring that old pids aren't zombies"
seconds_waited = 0
termed = false
while any_running pids
if seconds_waited >= seconds_to_wait
puts "waited #{seconds_waited} for old pids to exit.
they did not die gracefully.
terminating #{pids_string}"
if termed
puts "SIGTERM didn't work, killing #{pids_string}"
system("kill -9 #{pids_string} 2> /dev/null")
else
system("kill #{pids_string} 2> /dev/null")
termed = true
end
else
puts "old pids still alive.
waiting 2s and checking again"
sleep 2
nowpids = check_running || 0
seconds_waited = seconds_waited + 2
end
end
end

def write_server_state_file
return unless server_state_file
begin
File.open(server_state_file, 'w') do |f|
(1..nbproc).each do |i|
f.puts(unixsock('show servers state', i))
end
end
puts "reloaded haproxy on pids #{nowpids.join(', ')}"
else
puts 'haproxy is not running!'
end
end

def unixsock(command)
def unixsock(command, process)
output = []
runs = 0

begin
ctl = UNIXSocket.open(socket)
ctl = UNIXSocket.open(socket(process))
if ctl
ctl.write "#{command}\r\n"
else
puts "cannot talk to #{socket}"
puts "cannot talk to #{socket(process)}"
end
rescue Errno::EPIPE
ctl.close
Expand All @@ -65,7 +132,7 @@ def unixsock(command)
if runs < 4
retry
else
puts "the unix socket at #{socket} closed before we could complete this request"
puts "the unix socket at #{socket(process)} closed before we could complete this request"
exit
end
end
Expand All @@ -88,34 +155,39 @@ def usage
<<-USAGE
usage: #{$PROGRAM_NAME} <argument>
where <argument> can be:
start : start haproxy unless it is already running
stop : stop an existing haproxy
restart : immediately shutdown and restart
reload : gracefully terminate existing connections, reload #{config_path}
status : is haproxy running? on what ports per lsof?
configcheck : check #{config_path}
nagios : nagios-friendly status for running process and listener
cloudkick : cloudkick.com-friendly status and metric for connected users
show health : show status of all frontends and backend servers
show backends : show status of backend pools of servers
enable all <server> : re-enable a server previously in maint mode on multiple backends
disable all <server> : disable a server from every backend it exists
enable all EXCEPT <server> : like 'enable all', but re-enables every backend except for <server>
disable all EXCEPT <server> : like 'disable all', but disables every backend except for <server>
clear counters : clear max statistics counters (add 'all' for all counters)
help : this message
prompt : toggle interactive mode with prompt
quit : disconnect
show info : report information about the running process
show stat : report counters for each proxy and server
show errors : report last request and response errors for each proxy
show sess [id] : report the list of current sessions or dump this session
get weight : report a server's current weight
set weight : change a server's weight
set timeout : change a timeout setting
disable server : set a server in maintenance mode
enable server : re-enable a server that was previously in maintenance mode
version : version of this script
start : start haproxy unless it is already running
stop : stop an existing haproxy
restart : immediately shutdown and restart
reload : gracefully terminate existing connections, reload #{config_path}
status : is haproxy running? on what ports per lsof?
configcheck : check #{config_path}
nagios : nagios-friendly status for running process and listener
<proc?> show health : show status of all frontends and backend servers
<proc?> show backends : show status of backend pools of servers
<proc?> enable all <server> : re-enable a server previously in maint mode on multiple backends
<proc?> disable all <server> : disable a server from every backend it exists
<proc?> drain all <server> : drain a server from every backend it exists
<proc?> enable all EXCEPT <server> : like 'enable all', but re-enables every backend except for <server>
<proc?> disable all EXCEPT <server> : like 'disable all', but disables every backend except for <server>
<proc?> drain all EXCEPT <server> : like 'drain all', but drains every backend except for <server>
<proc?> clear counters : clear max statistics counters (add 'all' for all counters)
help : this message
prompt : toggle interactive mode with prompt
quit : disconnect
<proc?> show info : report information about the running process
<proc?> show stat : report counters for each proxy and server
<proc?> show errors : report last request and response errors for each proxy
<proc?> show sess [id] : report the list of current sessions or dump this session
<proc?> get weight : report a server's current weight
<proc?> set weight : change a server's weight
<proc?> set timeout : change a timeout setting
<proc?> disable server : set a server in maintenance mode
<proc?> enable server : re-enable a server that was previously in maintenance mode
version : version of this script

<proc?> is an optional numerical argument that selects the process number to target
- only applicable when nbproc > 1
- defaults to 1
USAGE
end
end
Loading