Merge pull request #5 from kcen/master

support ipv6 client ips

Author: tobyhs

lib/haproxy_log_parser/line.treetop

@@ -2,7 +2,7 @@ module HAProxyLogParser
   grammar Line
     rule line
       syslog_portion:([^\[]+ '[' integer ']: ')
-      client_ip:ip4_address ':' client_port:integer ' '
+      client_ip:ip_address ':' client_port:integer ' '
       '[' accept_date '] '
       suffix:(normal_suffix / error_suffix)
       "\n"?
@@ -38,8 +38,60 @@ module HAProxyLogParser
       ([0-9] 2..2 ':') 2..2 ([0-9] 2..2)
     end
 
+    rule ip_address
+      # Asserting port is a little hacky but required because of the way
+      # HAProxy writes IPv6 addresses in the logs. It uses a valid RFC5952
+      # method of IP:PORT, but this complicates things by having valid IP:PORT
+      # combos that are themselves valid IPv6 addresses.
+      # e.g. 1::2:123
+      ip4_address / ip6_address_assert_port
+    end
+
     rule ip4_address
-      ([0-9] 1..3 '.') 3..3 ([0-9] 1..3)
+      dec_octet '.' dec_octet '.' dec_octet '.' dec_octet
+    end
+
+    rule colon_port
+      ':' integer ' '
+    end
+
+    rule ip6_address_assert_port
+      # Taken from https://tools.ietf.org/html/rfc3986#section-3.2.2
+      # Performs look-ahead assertion on the port, assuming format IP:PORT
+                                      ( h16 ':' ) 6..6 ls32 &colon_port
+      /                          '::' ( h16 ':' ) 5..5 ls32 &colon_port
+      / ( h16                 )? '::' ( h16 ':' ) 4..4 ls32 &colon_port
+      / ( h16 ( ':' h16 ) ..1 )? '::' ( h16 ':' ) 3..3 ls32 &colon_port
+      / ( h16 ( ':' h16 ) ..2 )? '::' ( h16 ':' ) 2..2 ls32 &colon_port
+      / ( h16 ( ':' h16 ) ..3 )? '::'   h16 ':'        ls32 &colon_port
+      / ( h16 ( ':' h16 ) ..4 )? '::'                  ls32 &colon_port
+      / ( h16 ( ':' h16 ) ..5 )? '::'                  h16  &colon_port
+      / ( h16 ( ':' h16 ) ..6 )? '::'                       &colon_port
+    end
+
+    rule ls32
+      # least-significant 32 bits of address
+      ( h16 ':' h16 ) / ip4_address
+    end
+
+    rule h16
+      # 16 bits of address represented in hexadecimal
+      hex_digit 1..4
+    end
+
+    rule hex_digit
+      [a-f0-9A-F]
+    end
+
+    rule dec_octet
+      '25' [0-5]
+      /  '2' [0-4] dec_digit
+      /  '1' dec_digit 2..2
+      /  dec_digit 1..2
+    end
+
+    rule dec_digit
+      [0-9]
     end
 
     rule accept_date

spec/haproxy_log_parser_spec.rb

@@ -6,6 +6,9 @@
       map { |line| line.split(',', 2) }
   ]
 
+  TEST_IP_ADDRESSES = IO.readlines(File.join(File.dirname(__FILE__), 'test_ips.log')).
+      map { |line| line.chomp }
+
   describe '.parse' do
     it 'parses the good1 case correctly' do
       entry = HAProxyLogParser.parse(LINES['good1'])
@@ -69,6 +72,39 @@
       expect(entry.http_request).to eq('GET /images/image.gif HTTP/1.1')
     end
 
+    TEST_IP_ADDRESSES.each do |test_ip|
+      it "parses generic case with test IP #{test_ip} correctly" do
+        entry = HAProxyLogParser.parse(LINES['generic_ip_test'].sub(/IPADDRESS/, test_ip))
+        expect(entry.client_ip).to eq(test_ip)
+        expect(entry.client_port).to eq(50679)
+        expect(entry.accept_date).to eq(Time.local(2012, 5, 21, 1, 35, 46, 146))
+        expect(entry.frontend_name).to eq('webapp')
+        expect(entry).to_not be_ssl
+        expect(entry.backend_name).to eq('webapp_backend')
+        expect(entry.server_name).to eq('web09')
+        expect(entry.tq).to eq(27)
+        expect(entry.tw).to eq(0)
+        expect(entry.tc).to eq(1)
+        expect(entry.tr).to eq(0)
+        expect(entry.tt).to eq(217)
+        expect(entry.status_code).to eq(200)
+        expect(entry.bytes_read).to eq(1367)
+        expect(entry.captured_request_cookie).to eq({'session' => 'abc'})
+        expect(entry.captured_response_cookie).to eq({'session' => 'xyz'})
+        expect(entry.termination_state).to eq('----')
+        expect(entry.actconn).to eq(600)
+        expect(entry.feconn).to eq(529)
+        expect(entry.beconn).to eq(336)
+        expect(entry.srv_conn).to eq(158)
+        expect(entry.retries).to eq(0)
+        expect(entry.srv_queue).to eq(0)
+        expect(entry.backend_queue).to eq(0)
+        expect(entry.captured_request_headers).to eq(['|| {5F41}', 'http://google.com/', ''])
+        expect(entry.captured_response_headers).to eq([])
+        expect(entry.http_request).to eq('GET /images/image.gif HTTP/1.1')
+      end
+    end
+
     it 'parses connection error lines correctly' do
       entry = HAProxyLogParser.parse(LINES['error1'])
       expect(entry.client_ip).to eq('127.0.0.1')

spec/sample.log

@@ -1,3 +1,4 @@
 good1,Aug  9 20:30:46 localhost haproxy[2022]: 10.0.8.2:34028 [09/Aug/2011:20:30:46.429] proxy-out~ proxy-out/cache1 1/0/2/126/+128 301 +223 - - LR-- 617/523/336/168/0 0/0 {www.sytadin.equipement.gouv.fr||http://trafic.1wt.eu/} {Apache|230|||http://www.sytadin.} "GET / HTTP/1.1"
 good2,May 21 01:35:46 10.18.237.5 haproxy[26747]: 192.168.1.215:50679 [21/May/2012:01:35:46.146] webapp webapp_backend/web09 27/0/1/0/217 200 1367 session=abc session=xyz ---- 600/529/336/158/0 0/0 {#7C#7C #7B5F41#7D|http://google.com/|} "GET /images/image.gif HTTP/1.1"
+generic_ip_test,May 21 01:35:46 10.18.237.5 haproxy[26747]: IPADDRESS:50679 [21/May/2012:01:35:46.146] webapp webapp_backend/web09 27/0/1/0/217 200 1367 session=abc session=xyz ---- 600/529/336/158/0 0/0 {#7C#7C #7B5F41#7D|http://google.com/|} "GET /images/image.gif HTTP/1.1"
 error1,Dec  3 18:27:14 localhost haproxy[6103]: 127.0.0.1:56059 [03/Dec/2012:17:35:10.380] frt/f1: Connection error during SSL handshake

spec/test_ips.log

@@ -0,0 +1,16 @@
+12::3a:829
+123.41.31.2
+127.0.0.1
+2.3.4.50
+2001:db8::42:8329
+::ffff:192.0.2.128
+2001:0db8:85a3:08d3:1319:8a2e:0370:7334
+201:0db8:85a3::1319:8a2e:0370:7344
+2001:0DB8:0::0:1428:57ab
+2001:0DB8::1428:57ab
+::ffff:192.168.89.9
+::ffff:c0a8:5909
+::1.2.3.4
+::c0a8:5909
+::192.0.2.128
+1::2:123