multiline_test.go

package tail

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"github.com/influxdata/telegraf/config"
)

func TestMultilineConfigOK(t *testing.T) {
	c := &multilineConfig{
		Pattern:        ".*",
		MatchWhichLine: previous,
	}

	_, err := c.newMultiline()

	require.NoError(t, err, "Configuration was OK.")
}

func TestMultilineConfigError(t *testing.T) {
	c := &multilineConfig{
		Pattern:        "\xA0",
		MatchWhichLine: previous,
	}

	_, err := c.newMultiline()

	require.Error(t, err, "The pattern was invalid")
}

func TestMultilineConfigTimeoutSpecified(t *testing.T) {
	duration := config.Duration(10 * time.Second)
	c := &multilineConfig{
		Pattern:        ".*",
		MatchWhichLine: previous,
		Timeout:        &duration,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")

	require.Equal(t, duration, *m.config.Timeout)
}

func TestMultilineConfigDefaultTimeout(t *testing.T) {
	duration := config.Duration(5 * time.Second)
	c := &multilineConfig{
		Pattern:        ".*",
		MatchWhichLine: previous,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")

	require.Equal(t, duration, *m.config.Timeout)
}

func TestMultilineIsEnabled(t *testing.T) {
	c := &multilineConfig{
		Pattern:        ".*",
		MatchWhichLine: previous,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")

	isEnabled := m.isEnabled()

	require.True(t, isEnabled, "Should have been enabled")
}

func TestMultilineIsDisabled(t *testing.T) {
	c := &multilineConfig{
		MatchWhichLine: previous,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")

	isEnabled := m.isEnabled()

	require.False(t, isEnabled, "Should have been disabled")
}

func TestMultilineFlushEmpty(t *testing.T) {
	var buffer bytes.Buffer
	text := flush(&buffer)

	require.Empty(t, text)
}

func TestMultilineFlush(t *testing.T) {
	var buffer bytes.Buffer
	buffer.WriteString("foo")

	text := flush(&buffer)
	require.Equal(t, "foo", text)
	require.Zero(t, buffer.Len())
}

func TestMultiLineProcessLinePrevious(t *testing.T) {
	c := &multilineConfig{
		Pattern:        "^=>",
		MatchWhichLine: previous,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")
	var buffer bytes.Buffer

	text := m.processLine("1", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("=>2", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("=>3", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("4", &buffer)
	require.Equal(t, "1=>2=>3", text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("5", &buffer)
	require.Equal(t, "4", text)
	require.Equal(t, "5", buffer.String())
}

func TestMultiLineProcessLineNext(t *testing.T) {
	c := &multilineConfig{
		Pattern:        "=>$",
		MatchWhichLine: next,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")
	var buffer bytes.Buffer

	text := m.processLine("1=>", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("2=>", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("3=>", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("4", &buffer)
	require.Equal(t, "1=>2=>3=>4", text)
	require.Zero(t, buffer.Len())

	text = m.processLine("5", &buffer)
	require.Equal(t, "5", text)
	require.Zero(t, buffer.Len())
}

func TestMultiLineMatchStringWithInvertMatchFalse(t *testing.T) {
	c := &multilineConfig{
		Pattern:        "=>$",
		MatchWhichLine: next,
		InvertMatch:    false,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")

	matches1 := m.matchString("t=>")
	matches2 := m.matchString("t")

	require.True(t, matches1)
	require.False(t, matches2)
}

func TestMultiLineMatchStringWithInvertTrue(t *testing.T) {
	c := &multilineConfig{
		Pattern:        "=>$",
		MatchWhichLine: next,
		InvertMatch:    true,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")

	matches1 := m.matchString("t=>")
	matches2 := m.matchString("t")

	require.False(t, matches1)
	require.True(t, matches2)
}

func TestMultilineWhat(t *testing.T) {
	var w1 multilineMatchWhichLine
	require.NoError(t, w1.UnmarshalTOML([]byte(`"previous"`)))
	require.Equal(t, previous, w1)

	var w2 multilineMatchWhichLine
	require.NoError(t, w2.UnmarshalTOML([]byte(`previous`)))
	require.Equal(t, previous, w2)

	var w3 multilineMatchWhichLine
	require.NoError(t, w3.UnmarshalTOML([]byte(`'previous'`)))
	require.Equal(t, previous, w3)

	var w4 multilineMatchWhichLine
	require.NoError(t, w4.UnmarshalTOML([]byte(`"next"`)))
	require.Equal(t, next, w4)

	var w5 multilineMatchWhichLine
	require.NoError(t, w5.UnmarshalTOML([]byte(`next`)))
	require.Equal(t, next, w5)

	var w6 multilineMatchWhichLine
	require.NoError(t, w6.UnmarshalTOML([]byte(`'next'`)))
	require.Equal(t, next, w6)

	var w7 multilineMatchWhichLine
	require.Error(t, w7.UnmarshalTOML([]byte(`nope`)))
	require.Equal(t, multilineMatchWhichLine(-1), w7)
}

func TestMultilineQuoted(t *testing.T) {
	tests := []struct {
		name      string
		quotation string
		quote     string
		filename  string
	}{
		{
			name:      "single-quotes",
			quotation: "single-quotes",
			quote:     `'`,
			filename:  "multiline_quoted_single.csv",
		},
		{
			name:      "double-quotes",
			quotation: "double-quotes",
			quote:     `"`,
			filename:  "multiline_quoted_double.csv",
		},
		{
			name:      "backticks",
			quotation: "backticks",
			quote:     "`",
			filename:  "multiline_quoted_backticks.csv",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			expected := []string{
				`1660819827410,1,some text without quotes,A`,
				fmt.Sprintf("1660819827411,1,%ssome text all quoted%s,A", tt.quote, tt.quote),
				fmt.Sprintf("1660819827412,1,%ssome text all quoted\nbut wrapped%s,A", tt.quote, tt.quote),
				fmt.Sprintf("1660819827420,2,some text with %squotes%s,B", tt.quote, tt.quote),
				"1660819827430,3,some text with 'multiple \"quotes\" in `one` line',C",
				fmt.Sprintf("1660819827440,4,some multiline text with %squotes\n", tt.quote) +
					fmt.Sprintf("spanning \\%smultiple\\%s\n", tt.quote, tt.quote) +
					fmt.Sprintf("lines%s but do not %send\ndirectly%s,D", tt.quote, tt.quote, tt.quote),
				fmt.Sprintf("1660819827450,5,all of %sthis%s should %sbasically%s work...,E", tt.quote, tt.quote, tt.quote, tt.quote),
			}

			c := &multilineConfig{
				MatchWhichLine:  next,
				Quotation:       tt.quotation,
				PreserveNewline: true,
			}
			m, err := c.newMultiline()
			require.NoError(t, err)

			f, err := os.Open(filepath.Join("testdata", tt.filename))
			require.NoError(t, err)

			scanner := bufio.NewScanner(f)

			var buffer bytes.Buffer
			var result []string
			for scanner.Scan() {
				line := scanner.Text()

				text := m.processLine(line, &buffer)
				if text == "" {
					continue
				}
				result = append(result, text)
			}
			if text := flush(&buffer); text != "" {
				result = append(result, text)
			}

			require.EqualValues(t, expected, result)
		})
	}
}

func TestMultilineQuotedError(t *testing.T) {
	tests := []struct {
		name      string
		filename  string
		quotation string
		quote     string
		expected  []string
	}{
		{
			name:      "messed up quoting",
			filename:  "multiline_quoted_messed_up.csv",
			quotation: "single-quotes",
			quote:     `'`,
			expected: []string{
				"1660819827410,1,some text without quotes,A",
				"1660819827411,1,'some text all quoted,A\n1660819827412,1,'some text all quoted",
				"but wrapped,A"},
		},
		{
			name:      "missing closing quote",
			filename:  "multiline_quoted_missing_close.csv",
			quotation: "single-quotes",
			quote:     `'`,
			expected:  []string{"1660819827411,2,'some text all quoted,B\n1660819827410,1,some text without quotes,A"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			c := &multilineConfig{
				MatchWhichLine:  next,
				Quotation:       tt.quotation,
				PreserveNewline: true,
			}
			m, err := c.newMultiline()
			require.NoError(t, err)

			f, err := os.Open(filepath.Join("testdata", tt.filename))
			require.NoError(t, err)

			scanner := bufio.NewScanner(f)

			var buffer bytes.Buffer
			var result []string
			for scanner.Scan() {
				line := scanner.Text()

				text := m.processLine(line, &buffer)
				if text == "" {
					continue
				}
				result = append(result, text)
			}
			if text := flush(&buffer); text != "" {
				result = append(result, text)
			}

			require.EqualValues(t, tt.expected, result)
		})
	}
}

func TestMultilineNewline(t *testing.T) {
	tests := []struct {
		name     string
		filename string
		cfg      *multilineConfig
		expected []string
	}{
		{
			name: "do not preserve newline",
			cfg: &multilineConfig{
				Pattern:     `\[[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4}\]`,
				InvertMatch: true,
			},
			filename: "test_multiline.log",
			expected: []string{
				`[04/Jun/2016:12:41:45 +0100] DEBUG HelloExample: This is debug`,
				`[04/Jun/2016:12:41:48 +0100] INFO HelloExample: This is info`,
				"[04/Jun/2016:12:41:46 +0100] ERROR HelloExample: Sorry, something wrong! " +
					"java.lang.ArithmeticException: / by zero" +
					"\tat com.foo.HelloExample2.divide(HelloExample2.java:24)" +
					"\tat com.foo.HelloExample2.main(HelloExample2.java:14)",
				`[04/Jun/2016:12:41:48 +0100] WARN HelloExample: This is warn`,
			},
		},
		{
			name: "preserve newline",
			cfg: &multilineConfig{
				Pattern:         `\[[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4}\]`,
				InvertMatch:     true,
				PreserveNewline: true,
			},
			filename: "test_multiline.log",
			expected: []string{
				`[04/Jun/2016:12:41:45 +0100] DEBUG HelloExample: This is debug`,
				`[04/Jun/2016:12:41:48 +0100] INFO HelloExample: This is info`,
				`[04/Jun/2016:12:41:46 +0100] ERROR HelloExample: Sorry, something wrong!` + ` ` + `
java.lang.ArithmeticException: / by zero
	at com.foo.HelloExample2.divide(HelloExample2.java:24)
	at com.foo.HelloExample2.main(HelloExample2.java:14)`,
				`[04/Jun/2016:12:41:48 +0100] WARN HelloExample: This is warn`,
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			m, err := tt.cfg.newMultiline()
			require.NoError(t, err)

			f, err := os.Open(filepath.Join("testdata", tt.filename))
			require.NoError(t, err)

			scanner := bufio.NewScanner(f)

			var buffer bytes.Buffer
			var result []string
			for scanner.Scan() {
				line := scanner.Text()

				text := m.processLine(line, &buffer)
				if text == "" {
					continue
				}
				result = append(result, text)
			}
			if text := flush(&buffer); text != "" {
				result = append(result, text)
			}

			require.EqualValues(t, tt.expected, result)
		})
	}
}

func TestMultiLineQuotedAndPattern(t *testing.T) {
	c := &multilineConfig{
		Pattern:         "=>$",
		MatchWhichLine:  next,
		Quotation:       "double-quotes",
		PreserveNewline: true,
	}
	m, err := c.newMultiline()
	require.NoError(t, err, "Configuration was OK.")
	var buffer bytes.Buffer

	text := m.processLine("1=>", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("2=>", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine(`"a quoted`, &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine(`multiline string"=>`, &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("3=>", &buffer)
	require.Empty(t, text)
	require.NotZero(t, buffer.Len())

	text = m.processLine("4", &buffer)
	require.Equal(t, "1=>\n2=>\n\"a quoted\nmultiline string\"=>\n3=>\n4", text)
	require.Zero(t, buffer.Len())

	text = m.processLine("5", &buffer)
	require.Equal(t, "5", text)
	require.Zero(t, buffer.Len())
}