From cd0797b5db820178a5d69b1670e8d6e159dcfb90 Mon Sep 17 00:00:00 2001 From: Stephane Bortzmeyer Date: Sat, 15 Jul 2017 15:48:00 +0200 Subject: [PATCH 1/4] First version producing a correct CBOR file (with dummy data) --- CdnsFormat.go | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 CdnsFormat.go diff --git a/CdnsFormat.go b/CdnsFormat.go new file mode 100644 index 0000000..c63a4a3 --- /dev/null +++ b/CdnsFormat.go @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2013-2014 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "bytes" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/miekg/dns" +) + +const yamlTimeFormat = "2006-01-02 15:04:05.999999999" + +func yamlConvertMessage(m *Message, s *bytes.Buffer) { + s.WriteString(fmt.Sprint(" type: ", m.Type, "\n")) + + if m.QueryTimeSec != nil && m.QueryTimeNsec != nil { + t := time.Unix(int64(*m.QueryTimeSec), int64(*m.QueryTimeNsec)).UTC() + s.WriteString(fmt.Sprint(" query_time: !!timestamp ", t.Format(yamlTimeFormat), "\n")) + } + + if m.ResponseTimeSec != nil && m.ResponseTimeNsec != nil { + t := time.Unix(int64(*m.ResponseTimeSec), int64(*m.ResponseTimeNsec)).UTC() + s.WriteString(fmt.Sprint(" response_time: !!timestamp ", t.Format(yamlTimeFormat), "\n")) + } + + if m.SocketFamily != nil { + s.WriteString(fmt.Sprint(" socket_family: ", m.SocketFamily, "\n")) + } + + if m.SocketProtocol != nil { + s.WriteString(fmt.Sprint(" socket_protocol: ", m.SocketProtocol, "\n")) + } + + if m.QueryAddress != nil { + s.WriteString(fmt.Sprint(" query_address: ", net.IP(m.QueryAddress), "\n")) + } + + if m.ResponseAddress != nil { + s.WriteString(fmt.Sprint(" response_address: ", net.IP(m.ResponseAddress), "\n")) + } + + if m.QueryPort != nil { + s.WriteString(fmt.Sprint(" query_port: ", *m.QueryPort, "\n")) + } + + if m.ResponsePort != nil { + s.WriteString(fmt.Sprint(" response_port: ", *m.ResponsePort, "\n")) + } + + if m.QueryZone != nil { + name, _, err := dns.UnpackDomainName(m.QueryZone, 0) + if err != nil { + s.WriteString(" # query_zone: parse failed\n") + } else { + s.WriteString(fmt.Sprint(" query_zone: ", strconv.Quote(name), "\n")) + } + } + + if m.QueryMessage != nil { + msg := new(dns.Msg) + err := msg.Unpack(m.QueryMessage) + if err != nil { + s.WriteString(" # query_message: parse failed\n") + } else { + s.WriteString(" query_message: |\n") + s.WriteString(" " + strings.Replace(strings.TrimSpace(msg.String()), "\n", "\n ", -1) + "\n") + } + } + if m.ResponseMessage != nil { + msg := new(dns.Msg) + err := msg.Unpack(m.ResponseMessage) + if err != nil { + s.WriteString(fmt.Sprint(" # response_message: parse failed: ", err, "\n")) + } else { + s.WriteString(" response_message: |\n") + s.WriteString(" " + strings.Replace(strings.TrimSpace(msg.String()), "\n", "\n ", -1) + "\n") + } + } + s.WriteString("---\n") +} + +func YamlFormat(dt *Dnstap) (out []byte, ok bool) { + var s bytes.Buffer + + s.WriteString(fmt.Sprint("type: ", dt.Type, "\n")) + if dt.Identity != nil { + s.WriteString(fmt.Sprint("identity: ", strconv.Quote(string(dt.Identity)), "\n")) + } + if dt.Version != nil { + s.WriteString(fmt.Sprint("version: ", strconv.Quote(string(dt.Version)), "\n")) + } + if *dt.Type == Dnstap_MESSAGE { + s.WriteString("message:\n") + yamlConvertMessage(dt.Message, &s) + } + return s.Bytes(), true +} From 0bc85954bb4c956e19d888e0f4f77459f5b1656b Mon Sep 17 00:00:00 2001 From: Stephane Bortzmeyer Date: Sat, 15 Jul 2017 15:50:52 +0200 Subject: [PATCH 2/4] Other files modified for CBOR/C-DNS production --- CdnsFormat.go | 250 +++++++++++++++++++++++++++++++++++-------------- TextOutput.go | 28 +++++- dnstap/main.go | 20 ++-- 3 files changed, 215 insertions(+), 83 deletions(-) diff --git a/CdnsFormat.go b/CdnsFormat.go index c63a4a3..31c3827 100644 --- a/CdnsFormat.go +++ b/CdnsFormat.go @@ -19,98 +19,204 @@ package dnstap import ( "bytes" "fmt" - "net" - "strconv" - "strings" - "time" - - "github.com/miekg/dns" + "os" ) -const yamlTimeFormat = "2006-01-02 15:04:05.999999999" - -func yamlConvertMessage(m *Message, s *bytes.Buffer) { - s.WriteString(fmt.Sprint(" type: ", m.Type, "\n")) +var ( + first bool = true +) - if m.QueryTimeSec != nil && m.QueryTimeNsec != nil { - t := time.Unix(int64(*m.QueryTimeSec), int64(*m.QueryTimeNsec)).UTC() - s.WriteString(fmt.Sprint(" query_time: !!timestamp ", t.Format(yamlTimeFormat), "\n")) +func cborByteString(s []byte) []byte { + var ( + r bytes.Buffer + ) + if len(s) > 23 { + fmt.Fprintf(os.Stderr, "Long byte strings not yet supported TODO\n") + os.Exit(1) } + r.WriteByte(64 + byte(len(s))) // 64 = Major type 2 + r.Write(s) + return r.Bytes() +} - if m.ResponseTimeSec != nil && m.ResponseTimeNsec != nil { - t := time.Unix(int64(*m.ResponseTimeSec), int64(*m.ResponseTimeNsec)).UTC() - s.WriteString(fmt.Sprint(" response_time: !!timestamp ", t.Format(yamlTimeFormat), "\n")) +func cborString(s string) []byte { + var ( + r bytes.Buffer + ) + if len(s) > 23 { + fmt.Fprintf(os.Stderr, "Long strings not yet supported TODO\n") + os.Exit(1) } + r.WriteByte(96 + byte(len(s))) // 96 = Major type 3 + r.WriteString(s) + return r.Bytes() +} - if m.SocketFamily != nil { - s.WriteString(fmt.Sprint(" socket_family: ", m.SocketFamily, "\n")) +func cborInteger(i int) []byte { + var ( + r bytes.Buffer + ) + if i < 0 { + fmt.Fprintf(os.Stderr, "Negative integers not yet supported TODO\n") + os.Exit(1) } - - if m.SocketProtocol != nil { - s.WriteString(fmt.Sprint(" socket_protocol: ", m.SocketProtocol, "\n")) + if i > 23 { + fmt.Fprintf(os.Stderr, "Big integers not yet supported TODO\n") + os.Exit(1) } + r.WriteByte(0 + byte(i)) // 0 = Major type 0 + return r.Bytes() +} - if m.QueryAddress != nil { - s.WriteString(fmt.Sprint(" query_address: ", net.IP(m.QueryAddress), "\n")) +func cborArray(size uint) []byte { + var ( + r bytes.Buffer + ) + if size > 23 { + fmt.Fprintf(os.Stderr, "Big arrays not yet supported TODO\n") + os.Exit(1) } + r.WriteByte(128 + byte(size)) // 128 = Major type 3 + return r.Bytes() +} - if m.ResponseAddress != nil { - s.WriteString(fmt.Sprint(" response_address: ", net.IP(m.ResponseAddress), "\n")) - } +func cborArrayIndef() []byte { + var ( + r bytes.Buffer + ) + r.WriteByte(128 + 31) // 128 = Major type 3, 31 = indefinite + return r.Bytes() +} - if m.QueryPort != nil { - s.WriteString(fmt.Sprint(" query_port: ", *m.QueryPort, "\n")) +// TODO indefinite length maps +func cborMap(size uint) []byte { + var ( + r bytes.Buffer + ) + if size > 23 { + fmt.Fprintf(os.Stderr, "Big maps not yet supported TODO\n") + os.Exit(1) } + r.WriteByte(160 + byte(size)) // 160 = Major type 4 + return r.Bytes() +} - if m.ResponsePort != nil { - s.WriteString(fmt.Sprint(" response_port: ", *m.ResponsePort, "\n")) - } +func cborBreak() []byte { + var ( + r bytes.Buffer + ) + r.WriteByte(224 + 31) // 224 = Major type 7 + 31 = mandatory for break + return r.Bytes() +} - if m.QueryZone != nil { - name, _, err := dns.UnpackDomainName(m.QueryZone, 0) - if err != nil { - s.WriteString(" # query_zone: parse failed\n") - } else { - s.WriteString(fmt.Sprint(" query_zone: ", strconv.Quote(name), "\n")) - } +func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { + var ( + s bytes.Buffer + dummy []byte + ) + + dummy = make([]byte, 16) + if first { + s.Write(cborArray(3)) + // File type ID + s.Write(cborString("C-DNS")) + // Preamble + s.Write(cborMap(3)) + // Major version + s.Write(cborInteger(0)) + s.Write(cborInteger(0)) + // Minor version + s.Write(cborInteger(1)) + s.Write(cborInteger(5)) + // Generator ID + s.Write(cborInteger(4)) + s.Write(cborString("IETF 99 hackathon")) + // Blocks + s.Write(cborArrayIndef()) + first = false } - if m.QueryMessage != nil { - msg := new(dns.Msg) - err := msg.Unpack(m.QueryMessage) - if err != nil { - s.WriteString(" # query_message: parse failed\n") - } else { - s.WriteString(" query_message: |\n") - s.WriteString(" " + strings.Replace(strings.TrimSpace(msg.String()), "\n", "\n ", -1) + "\n") - } - } - if m.ResponseMessage != nil { - msg := new(dns.Msg) - err := msg.Unpack(m.ResponseMessage) - if err != nil { - s.WriteString(fmt.Sprint(" # response_message: parse failed: ", err, "\n")) - } else { - s.WriteString(" response_message: |\n") - s.WriteString(" " + strings.Replace(strings.TrimSpace(msg.String()), "\n", "\n ", -1) + "\n") - } - } - s.WriteString("---\n") + // Write a block for each message + s.Write(cborMap(3)) + // Block preamble + s.Write(cborInteger(0)) + s.Write(cborMap(1)) + s.Write(cborInteger(1)) + s.Write(cborArray(2)) + s.Write(cborInteger(0)) // TODO real seconds + s.Write(cborInteger(0)) // TODO real microseconds + // Block tables + s.Write(cborInteger(2)) + s.Write(cborMap(4)) + // IP addresses + s.Write(cborInteger(0)) + s.Write(cborArray(1)) + s.Write(cborByteString(dummy)) // TODO put the real IP address + // Class type + s.Write(cborInteger(1)) + s.Write(cborArray(1)) + s.Write(cborMap(2)) + s.Write(cborInteger(0)) + s.Write(cborInteger(1)) // Class IN + s.Write(cborInteger(1)) + s.Write(cborInteger(15)) // Type MX + // Name rdata + s.Write(cborInteger(2)) + s.Write(cborArray(1)) + b := make([]byte, 5) + b[0] = 3 + b[1] = 'c' + b[2] = 'o' + b[3] = 'm' + b[4] = 0 + s.Write(cborByteString(b)) // TODO real data + // Query sig + s.Write(cborInteger(3)) + s.Write(cborArray(1)) + s.Write(cborMap(5)) + // Server address index + s.Write(cborInteger(0)) + s.Write(cborInteger(1)) + // Server port + s.Write(cborInteger(1)) + s.Write(cborInteger(11)) // TODO real port + // Transport flags + s.Write(cborInteger(2)) + s.Write(cborInteger(2)) // IPv6. TODO: use real ones + // QR sig flags + s.Write(cborInteger(3)) + s.Write(cborInteger(0)) + // QR DNS flags + s.Write(cborInteger(5)) + s.Write(cborInteger(0)) + // End of block tables + // Queries/Responses + s.Write(cborInteger(3)) + s.Write(cborArray(1)) + s.Write(cborMap(5)) + // Time + s.Write(cborInteger(0)) + s.Write(cborInteger(0)) + // Client address index + s.Write(cborInteger(2)) + s.Write(cborInteger(1)) + // Client port + s.Write(cborInteger(3)) + s.Write(cborInteger(10)) // TODO put the real port + // Transaction ID + s.Write(cborInteger(4)) + s.Write(cborInteger(6)) // TODO put the real ID + // Query signature index + s.Write(cborInteger(5)) + s.Write(cborInteger(1)) + // End of block + + fmt.Fprintf(os.Stderr, "%d bytes emitted\n", len(s.Bytes())) + return s.Bytes(), true } -func YamlFormat(dt *Dnstap) (out []byte, ok bool) { +func CdnsFinish(dt *Dnstap) (out []byte, ok bool) { var s bytes.Buffer - - s.WriteString(fmt.Sprint("type: ", dt.Type, "\n")) - if dt.Identity != nil { - s.WriteString(fmt.Sprint("identity: ", strconv.Quote(string(dt.Identity)), "\n")) - } - if dt.Version != nil { - s.WriteString(fmt.Sprint("version: ", strconv.Quote(string(dt.Version)), "\n")) - } - if *dt.Type == Dnstap_MESSAGE { - s.WriteString("message:\n") - yamlConvertMessage(dt.Message, &s) - } + s.Write(cborBreak()) // End of the blocks array return s.Bytes(), true } diff --git a/TextOutput.go b/TextOutput.go index 5fa2c00..69ffb60 100644 --- a/TextOutput.go +++ b/TextOutput.go @@ -26,32 +26,39 @@ import ( ) type TextFormatFunc func(*Dnstap) ([]byte, bool) +type TextFinishFunc func(*Dnstap) ([]byte, bool) + +func DoNothing(dt *Dnstap) (out []byte, ok bool) { + return nil, false +} type TextOutput struct { format TextFormatFunc + finish TextFinishFunc outputChannel chan []byte wait chan bool writer *bufio.Writer } -func NewTextOutput(writer io.Writer, format TextFormatFunc) (o *TextOutput) { +func NewTextOutput(writer io.Writer, format TextFormatFunc, finish TextFinishFunc) (o *TextOutput) { o = new(TextOutput) o.format = format + o.finish = finish o.outputChannel = make(chan []byte, outputChannelSize) o.writer = bufio.NewWriter(writer) o.wait = make(chan bool) return } -func NewTextOutputFromFilename(fname string, format TextFormatFunc) (o *TextOutput, err error) { +func NewTextOutputFromFilename(fname string, format TextFormatFunc, finish TextFinishFunc) (o *TextOutput, err error) { if fname == "" || fname == "-" { - return NewTextOutput(os.Stdout, format), nil + return NewTextOutput(os.Stdout, format, finish), nil } writer, err := os.Create(fname) if err != nil { return } - return NewTextOutput(writer, format), nil + return NewTextOutput(writer, format, finish), nil } func (o *TextOutput) GetOutputChannel() chan []byte { @@ -70,7 +77,8 @@ func (o *TextOutput) RunOutputLoop() { log.Fatalf("dnstap.TextOutput: text format function failed\n") break } - if _, err := o.writer.Write(buf); err != nil { + _, err := o.writer.Write(buf) + if err != nil { log.Fatalf("dnstap.TextOutput: write failed: %s\n", err) break } @@ -80,6 +88,16 @@ func (o *TextOutput) RunOutputLoop() { } func (o *TextOutput) Close() { + dt := &Dnstap{} + buf, ok := o.finish(dt) + if !ok { + log.Fatalf("dnstap.TextOutput: text finish function failed\n") + } + _, err := o.writer.Write(buf) + if err != nil { + log.Fatalf("dnstap.TextOutput: write failed: %s\n", err) + } + o.writer.Flush() close(o.outputChannel) <-o.wait o.writer.Flush() diff --git a/dnstap/main.go b/dnstap/main.go index 5d20cb2..a22fb7f 100644 --- a/dnstap/main.go +++ b/dnstap/main.go @@ -25,7 +25,7 @@ import ( "runtime" "syscall" - "github.com/dnstap/golang-dnstap" + "github.com/bortzmeyer/golang-dnstap" ) var ( @@ -34,6 +34,7 @@ var ( flagWriteFile = flag.String("w", "-", "write output to file") flagQuietText = flag.Bool("q", false, "use quiet text output") flagYamlText = flag.Bool("y", false, "use verbose YAML output") + flagCdnsText = flag.Bool("c", false, "use C-DNS output") ) func usage() { @@ -56,14 +57,16 @@ Quiet text output format mnemonics: `) } -func outputOpener(fname string, text, yaml bool) func() dnstap.Output { +func outputOpener(fname string, text, yaml bool, cdns bool) func() dnstap.Output { return func() dnstap.Output { var o dnstap.Output var err error if text { - o, err = dnstap.NewTextOutputFromFilename(fname, dnstap.TextFormat) + o, err = dnstap.NewTextOutputFromFilename(fname, dnstap.TextFormat, dnstap.DoNothing) } else if yaml { - o, err = dnstap.NewTextOutputFromFilename(fname, dnstap.YamlFormat) + o, err = dnstap.NewTextOutputFromFilename(fname, dnstap.YamlFormat, dnstap.DoNothing) + } else if cdns { + o, err = dnstap.NewTextOutputFromFilename(fname, dnstap.CdnsFormat, dnstap.CdnsFinish) } else { o, err = dnstap.NewFrameStreamOutputFromFilename(fname) } @@ -121,8 +124,13 @@ func main() { os.Exit(1) } + if *flagYamlText && *flagCdnsText { + fmt.Fprintf(os.Stderr, "dnstap: Error: YAML or C-DNS but not both.\n") + os.Exit(1) + } + if *flagWriteFile == "-" { - if *flagQuietText == false && *flagYamlText == false { + if *flagQuietText == false && *flagYamlText == false && !*flagCdnsText { *flagQuietText = true } } @@ -134,7 +142,7 @@ func main() { // Start the output loop. output := make(chan []byte, 1) - opener := outputOpener(*flagWriteFile, *flagQuietText, *flagYamlText) + opener := outputOpener(*flagWriteFile, *flagQuietText, *flagYamlText, *flagCdnsText) outDone := make(chan struct{}) go outputLoop(opener, output, outDone) From f7fba5c0bb339dba31820f8d0b932b1debb1d7e0 Mon Sep 17 00:00:00 2001 From: Stephane Bortzmeyer Date: Sat, 15 Jul 2017 21:06:05 +0200 Subject: [PATCH 3/4] Addresses and ports --- CdnsFormat.go | 231 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 148 insertions(+), 83 deletions(-) diff --git a/CdnsFormat.go b/CdnsFormat.go index 31c3827..fb997b6 100644 --- a/CdnsFormat.go +++ b/CdnsFormat.go @@ -18,6 +18,7 @@ package dnstap import ( "bytes" + "encoding/binary" "fmt" "os" ) @@ -54,17 +55,30 @@ func cborString(s string) []byte { func cborInteger(i int) []byte { var ( - r bytes.Buffer + r bytes.Buffer + tmp []byte ) if i < 0 { fmt.Fprintf(os.Stderr, "Negative integers not yet supported TODO\n") os.Exit(1) } - if i > 23 { - fmt.Fprintf(os.Stderr, "Big integers not yet supported TODO\n") - os.Exit(1) + if i <= 23 { + r.WriteByte(0 + byte(i)) // 0 = Major type 0 + } else { + if i > 65535 { + fmt.Fprintf(os.Stderr, "Big integers not yet supported TODO\n") + os.Exit(1) + } + if i <= 255 { + r.WriteByte(0 + 24) // 0 = Major type 0 , 24 = additional byte + r.WriteByte(byte(i)) + } else { + tmp = make([]byte, 2) + r.WriteByte(0 + 25) + binary.BigEndian.PutUint16(tmp, uint16(i)) + r.Write(tmp) + } } - r.WriteByte(0 + byte(i)) // 0 = Major type 0 return r.Bytes() } @@ -111,11 +125,15 @@ func cborBreak() []byte { func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { var ( - s bytes.Buffer - dummy []byte + s bytes.Buffer + clientAddr []byte = nil + serverAddr []byte = nil + dummy []byte = nil + sourcePort uint32 = 0 + destinationPort uint32 = 0 ) - dummy = make([]byte, 16) + // TODO : put dt.Type, dt.Identity and dt.Version somewhere if first { s.Write(cborArray(3)) // File type ID @@ -136,82 +154,129 @@ func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { first = false } - // Write a block for each message - s.Write(cborMap(3)) - // Block preamble - s.Write(cborInteger(0)) - s.Write(cborMap(1)) - s.Write(cborInteger(1)) - s.Write(cborArray(2)) - s.Write(cborInteger(0)) // TODO real seconds - s.Write(cborInteger(0)) // TODO real microseconds - // Block tables - s.Write(cborInteger(2)) - s.Write(cborMap(4)) - // IP addresses - s.Write(cborInteger(0)) - s.Write(cborArray(1)) - s.Write(cborByteString(dummy)) // TODO put the real IP address - // Class type - s.Write(cborInteger(1)) - s.Write(cborArray(1)) - s.Write(cborMap(2)) - s.Write(cborInteger(0)) - s.Write(cborInteger(1)) // Class IN - s.Write(cborInteger(1)) - s.Write(cborInteger(15)) // Type MX - // Name rdata - s.Write(cborInteger(2)) - s.Write(cborArray(1)) - b := make([]byte, 5) - b[0] = 3 - b[1] = 'c' - b[2] = 'o' - b[3] = 'm' - b[4] = 0 - s.Write(cborByteString(b)) // TODO real data - // Query sig - s.Write(cborInteger(3)) - s.Write(cborArray(1)) - s.Write(cborMap(5)) - // Server address index - s.Write(cborInteger(0)) - s.Write(cborInteger(1)) - // Server port - s.Write(cborInteger(1)) - s.Write(cborInteger(11)) // TODO real port - // Transport flags - s.Write(cborInteger(2)) - s.Write(cborInteger(2)) // IPv6. TODO: use real ones - // QR sig flags - s.Write(cborInteger(3)) - s.Write(cborInteger(0)) - // QR DNS flags - s.Write(cborInteger(5)) - s.Write(cborInteger(0)) - // End of block tables - // Queries/Responses - s.Write(cborInteger(3)) - s.Write(cborArray(1)) - s.Write(cborMap(5)) - // Time - s.Write(cborInteger(0)) - s.Write(cborInteger(0)) - // Client address index - s.Write(cborInteger(2)) - s.Write(cborInteger(1)) - // Client port - s.Write(cborInteger(3)) - s.Write(cborInteger(10)) // TODO put the real port - // Transaction ID - s.Write(cborInteger(4)) - s.Write(cborInteger(6)) // TODO put the real ID - // Query signature index - s.Write(cborInteger(5)) - s.Write(cborInteger(1)) - // End of block + if *dt.Type == Dnstap_MESSAGE { + m := dt.Message + // Write a block for each message. TODO: buffer for a few blocks + s.Write(cborMap(3)) + // Block preamble + s.Write(cborInteger(0)) + s.Write(cborMap(1)) + s.Write(cborInteger(1)) + s.Write(cborArray(2)) + s.Write(cborInteger(0)) // TODO real seconds + s.Write(cborInteger(0)) // TODO real microseconds + // Block tables + s.Write(cborInteger(2)) + s.Write(cborMap(4)) + // IP addresses. + s.Write(cborInteger(0)) + clientAddr = m.QueryAddress + serverAddr = m.ResponseAddress + if m.QueryPort != nil { + sourcePort = *m.QueryPort + } + if m.ResponsePort != nil { + destinationPort = *m.ResponsePort + } + if clientAddr == nil && serverAddr == nil { + fmt.Fprintf(os.Stderr, "No IP addresses at all in the message, I give in\n") + os.Exit(1) + } + s.Write(cborArray(2)) + if clientAddr != nil && serverAddr != nil { + s.Write(cborByteString(clientAddr)) + s.Write(cborByteString(serverAddr)) + } else { + switch *m.SocketFamily { + case SocketFamily_INET: + dummy = make([]byte, 4) + case SocketFamily_INET6: + dummy = make([]byte, 16) + default: + fmt.Fprintf(os.Stderr, "Unknown socket family %d\n", *m.SocketFamily) + os.Exit(1) + } + if clientAddr == nil { + s.Write(cborByteString(dummy)) + s.Write(cborByteString(serverAddr)) + } else { + s.Write(cborByteString(clientAddr)) + s.Write(cborByteString(dummy)) + } + } + // Class type + s.Write(cborInteger(1)) + s.Write(cborArray(1)) + s.Write(cborMap(2)) + s.Write(cborInteger(0)) + s.Write(cborInteger(15)) // Type MX + s.Write(cborInteger(1)) + s.Write(cborInteger(1)) // Class IN + // Name rdata + s.Write(cborInteger(2)) + s.Write(cborArray(1)) + b := make([]byte, 5) + b[0] = 3 + b[1] = 'c' + b[2] = 'o' + b[3] = 'm' + b[4] = 0 + s.Write(cborByteString(b)) // TODO real data + // Query sig + s.Write(cborInteger(3)) + s.Write(cborArray(1)) + s.Write(cborMap(5)) + // Server address index + s.Write(cborInteger(0)) + s.Write(cborInteger(2)) + // Server port + s.Write(cborInteger(1)) + s.Write(cborInteger(int(destinationPort))) + // Transport flags + s.Write(cborInteger(2)) + switch *m.SocketFamily { + case SocketFamily_INET: + s.Write(cborInteger(0)) + case SocketFamily_INET6: + s.Write(cborInteger(2)) + default: + fmt.Fprintf(os.Stderr, "Unknown socket family %d\n", *m.SocketFamily) + os.Exit(1) + } + // QR sig flags + s.Write(cborInteger(3)) + if m.QueryMessage != nil { + s.Write(cborInteger(1)) + } else { // A response + s.Write(cborInteger(2)) + } + // QR DNS flags + s.Write(cborInteger(5)) + s.Write(cborInteger(0)) + // End of block tables + // Queries/Responses + s.Write(cborInteger(3)) + s.Write(cborArray(1)) + s.Write(cborMap(5)) + // Time + s.Write(cborInteger(0)) + s.Write(cborInteger(0)) + // Client address index + s.Write(cborInteger(2)) + s.Write(cborInteger(1)) + // Client port + s.Write(cborInteger(3)) + s.Write(cborInteger(int(sourcePort))) + // Transaction ID + s.Write(cborInteger(4)) + s.Write(cborInteger(666)) // TODO put the real ID + // Query signature index + s.Write(cborInteger(5)) + s.Write(cborInteger(1)) + // End of block - fmt.Fprintf(os.Stderr, "%d bytes emitted\n", len(s.Bytes())) + fmt.Fprintf(os.Stderr, "%d bytes emitted\n", len(s.Bytes())) + } return s.Bytes(), true } From 7e1cf1867b32e424632137e517c888453bc86ed7 Mon Sep 17 00:00:00 2001 From: Stephane Bortzmeyer Date: Sun, 16 Jul 2017 13:04:42 +0200 Subject: [PATCH 4/4] Proper handling of QNAMEs, QTYPEs and query times --- CdnsFormat.go | 186 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 146 insertions(+), 40 deletions(-) diff --git a/CdnsFormat.go b/CdnsFormat.go index fb997b6..1b2bcd3 100644 --- a/CdnsFormat.go +++ b/CdnsFormat.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 by Farsight Security, Inc. + * Copyright (c) 2013-2017 by Farsight Security, Inc and AFNIC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,14 @@ * limitations under the License. */ +/* The C-DNS output was written by Stephane Bortzmeyer +/* with help from Jim Hague . It +/* follows the Internet-Draft +/* draft-ietf-dnsop-dns-capture-format-03. It is far from optimized: +/* most important, it creates one block for every request, thus +/* defeating the whole point of C-DNS (compression). Consider it as a +/* Proof-of-Concept. */ + package dnstap import ( @@ -21,6 +29,8 @@ import ( "encoding/binary" "fmt" "os" + + "github.com/miekg/dns" ) var ( @@ -31,11 +41,23 @@ func cborByteString(s []byte) []byte { var ( r bytes.Buffer ) - if len(s) > 23 { - fmt.Fprintf(os.Stderr, "Long byte strings not yet supported TODO\n") - os.Exit(1) + if len(s) <= 23 { + r.WriteByte(64 + byte(len(s))) // 64 = Major type 2 + } else { + if len(s) > 65535 { + fmt.Fprintf(os.Stderr, "Big byte strings not yet supported TODO\n") + os.Exit(1) + } + if len(s) <= 255 { + r.WriteByte(64 + 24) // 64 = Major type 2 , 24 = additional byte + r.WriteByte(byte(len(s))) + } else { + tmp := make([]byte, 2) + r.WriteByte(64 + 25) + binary.BigEndian.PutUint16(tmp, uint16(len(s))) + r.Write(tmp) + } } - r.WriteByte(64 + byte(len(s))) // 64 = Major type 2 r.Write(s) return r.Bytes() } @@ -44,11 +66,23 @@ func cborString(s string) []byte { var ( r bytes.Buffer ) - if len(s) > 23 { - fmt.Fprintf(os.Stderr, "Long strings not yet supported TODO\n") - os.Exit(1) + if len(s) <= 23 { + r.WriteByte(96 + byte(len(s))) // 96 = Major type 3 + } else { + if len(s) > 65535 { + fmt.Fprintf(os.Stderr, "Big strings not yet supported TODO\n") + os.Exit(1) + } + if len(s) <= 255 { + r.WriteByte(96 + 24) // 96 = Major type 3 , 24 = additional byte + r.WriteByte(byte(len(s))) + } else { + tmp := make([]byte, 2) + r.WriteByte(96 + 25) + binary.BigEndian.PutUint16(tmp, uint16(len(s))) + r.Write(tmp) + } } - r.WriteByte(96 + byte(len(s))) // 96 = Major type 3 r.WriteString(s) return r.Bytes() } @@ -66,17 +100,20 @@ func cborInteger(i int) []byte { r.WriteByte(0 + byte(i)) // 0 = Major type 0 } else { if i > 65535 { - fmt.Fprintf(os.Stderr, "Big integers not yet supported TODO\n") - os.Exit(1) - } - if i <= 255 { - r.WriteByte(0 + 24) // 0 = Major type 0 , 24 = additional byte - r.WriteByte(byte(i)) - } else { - tmp = make([]byte, 2) - r.WriteByte(0 + 25) - binary.BigEndian.PutUint16(tmp, uint16(i)) + tmp = make([]byte, 4) + r.WriteByte(0 + 26) + binary.BigEndian.PutUint32(tmp, uint32(i)) r.Write(tmp) + } else { + if i <= 255 { + r.WriteByte(0 + 24) // 0 = Major type 0 , 24 = additional byte + r.WriteByte(byte(i)) + } else { + tmp = make([]byte, 2) + r.WriteByte(0 + 25) + binary.BigEndian.PutUint16(tmp, uint16(i)) + r.Write(tmp) + } } } return r.Bytes() @@ -102,7 +139,6 @@ func cborArrayIndef() []byte { return r.Bytes() } -// TODO indefinite length maps func cborMap(size uint) []byte { var ( r bytes.Buffer @@ -115,6 +151,14 @@ func cborMap(size uint) []byte { return r.Bytes() } +func cborIndefMap() []byte { + var ( + r bytes.Buffer + ) + r.WriteByte(160 + 31) // 160 = Major type 4, 31 = indefinite + return r.Bytes() +} + func cborBreak() []byte { var ( r bytes.Buffer @@ -131,15 +175,15 @@ func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { dummy []byte = nil sourcePort uint32 = 0 destinationPort uint32 = 0 + msg *dns.Msg ) - // TODO : put dt.Type, dt.Identity and dt.Version somewhere if first { s.Write(cborArray(3)) // File type ID s.Write(cborString("C-DNS")) // Preamble - s.Write(cborMap(3)) + s.Write(cborMap(4)) // Major version s.Write(cborInteger(0)) s.Write(cborInteger(0)) @@ -148,7 +192,10 @@ func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { s.Write(cborInteger(5)) // Generator ID s.Write(cborInteger(4)) - s.Write(cborString("IETF 99 hackathon")) + s.Write(cborString(fmt.Sprintf("Experimental dnstap client, IETF 99 hackathon, data from %s", dt.Version))) + // Host ID + s.Write(cborInteger(5)) + s.Write(cborString(string(dt.Identity))) // Blocks s.Write(cborArrayIndef()) first = false @@ -156,18 +203,23 @@ func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { if *dt.Type == Dnstap_MESSAGE { m := dt.Message - // Write a block for each message. TODO: buffer for a few blocks + // Write a block for each message. + // TODO: buffer for a few blocks (quite complicated, needs indexing, breaks existing assumptions, etc) s.Write(cborMap(3)) // Block preamble s.Write(cborInteger(0)) s.Write(cborMap(1)) s.Write(cborInteger(1)) s.Write(cborArray(2)) - s.Write(cborInteger(0)) // TODO real seconds + if m.QueryTimeSec != nil { + s.Write(cborInteger(int(*m.QueryTimeSec))) + } else { + s.Write(cborInteger(0)) + } s.Write(cborInteger(0)) // TODO real microseconds // Block tables s.Write(cborInteger(2)) - s.Write(cborMap(4)) + s.Write(cborIndefMap()) // IP addresses. s.Write(cborInteger(0)) clientAddr = m.QueryAddress @@ -208,24 +260,51 @@ func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { s.Write(cborInteger(1)) s.Write(cborArray(1)) s.Write(cborMap(2)) + if m.QueryMessage != nil { + msg = new(dns.Msg) + err := msg.Unpack(m.QueryMessage) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot unpack a DNS query message: %s\n", err) + os.Exit(1) + } + } s.Write(cborInteger(0)) - s.Write(cborInteger(15)) // Type MX + if m.QueryMessage != nil { + s.Write(cborInteger(int(msg.Question[0].Qtype))) + } else { + s.Write(cborInteger(0)) + } s.Write(cborInteger(1)) s.Write(cborInteger(1)) // Class IN // Name rdata s.Write(cborInteger(2)) + qname := make([]byte, 256) + n := 0 + err := error(nil) + if m.QueryMessage != nil { + n, err = dns.PackDomainName(msg.Question[0].Name, qname, 0, nil, false) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot pack a domain name: %s\n", err) + os.Exit(1) + } + } else { + qname[0] = 0 + n = 0 + } + if m.ResponseMessage != nil { + msg = new(dns.Msg) + err := msg.Unpack(m.ResponseMessage) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot unpack a DNS response message: %s\n", err) + os.Exit(1) + } + } s.Write(cborArray(1)) - b := make([]byte, 5) - b[0] = 3 - b[1] = 'c' - b[2] = 'o' - b[3] = 'm' - b[4] = 0 - s.Write(cborByteString(b)) // TODO real data + s.Write(cborByteString(qname[0:n])) // Query sig s.Write(cborInteger(3)) s.Write(cborArray(1)) - s.Write(cborMap(5)) + s.Write(cborIndefMap()) // Server address index s.Write(cborInteger(0)) s.Write(cborInteger(2)) @@ -246,20 +325,44 @@ func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { // QR sig flags s.Write(cborInteger(3)) if m.QueryMessage != nil { - s.Write(cborInteger(1)) + s.Write(cborInteger(1 + 4)) // Bit 0 has-query, bit 2 has-question } else { // A response s.Write(cborInteger(2)) } // QR DNS flags s.Write(cborInteger(5)) s.Write(cborInteger(0)) - // End of block tables + // Query class type index + s.Write(cborInteger(7)) + s.Write(cborInteger(1)) + if m.QueryMessage != nil { + // QD count + s.Write(cborInteger(8)) + s.Write(cborInteger(len(msg.Question))) + } + s.Write(cborBreak()) // End of query signature + if m.QueryMessage != nil { // Useless (used only if there are several questions. TODO try to drop it + // Question list + s.Write(cborInteger(4)) + s.Write(cborArray(1)) + s.Write(cborArray(1)) + s.Write(cborInteger(1)) + // Question RR + s.Write(cborInteger(5)) + s.Write(cborArray(1)) + s.Write(cborMap(2)) + s.Write(cborInteger(0)) // Name + s.Write(cborInteger(1)) + s.Write(cborInteger(1)) // Class/type + s.Write(cborInteger(1)) + } + s.Write(cborBreak()) // End of block tables // Queries/Responses s.Write(cborInteger(3)) s.Write(cborArray(1)) - s.Write(cborMap(5)) + s.Write(cborIndefMap()) // Time - s.Write(cborInteger(0)) + s.Write(cborInteger(0)) // TODO real time s.Write(cborInteger(0)) // Client address index s.Write(cborInteger(2)) @@ -273,9 +376,12 @@ func CdnsFormat(dt *Dnstap) (out []byte, ok bool) { // Query signature index s.Write(cborInteger(5)) s.Write(cborInteger(1)) + // Query name index + s.Write(cborInteger(9)) + s.Write(cborInteger(1)) + s.Write(cborBreak()) // End of block - fmt.Fprintf(os.Stderr, "%d bytes emitted\n", len(s.Bytes())) } return s.Bytes(), true }